Add capacity overview with host resource history
This commit is contained in:
@@ -49,6 +49,10 @@ including SQLite metadata and published game-file storage.
|
||||
See [Native host management](docs/native-hosts.md) for installation, credentials,
|
||||
storage, failure recovery, API contracts and the live-host acceptance checklist.
|
||||
|
||||
**Capacity** shows available CPU, memory, ports, estimated additional servers,
|
||||
and the shared startup queue. Filter by host or game, inspect each host's limiting
|
||||
resource, and explore up to seven days of resource headroom.
|
||||
|
||||
Existing hosts need the current installer once to enable automatic updates.
|
||||
Generate it under **Agent access** and rerun it on the same host without
|
||||
`--reenroll`; identity and game data are preserved. Thereafter hosts check their
|
||||
|
||||
@@ -233,6 +233,29 @@ systemctl status electric-cherry-update.timer
|
||||
journalctl -u electric-cherry-update.service -n 100
|
||||
```
|
||||
|
||||
## Capacity overview
|
||||
|
||||
Open **Capacity** (`/capacity`) to see the same cached estimates used by collection
|
||||
previews. Available totals include hosts ready for the default runtime and latest
|
||||
files for the selected game. **All games** counts shared resources once, including
|
||||
hosts ready for either game; it is not a guarantee that every game fits every host.
|
||||
The host breakdown shows readiness, missing measurements, available ports, learned
|
||||
running/startup allowances, and the resource limiting additional servers.
|
||||
|
||||
Startup slots and queued jobs are fleet-wide, even with a host or game selected.
|
||||
Server estimates use running allowances; provisioning still checks the larger
|
||||
startup allowance before starting each job. Port availability uses each host's
|
||||
configured bind address. Queued unplaced jobs do not reserve physical resources.
|
||||
|
||||
The history graph offers the last hour, 24 hours, or seven days of measured CPU
|
||||
and memory headroom, after the usual OS reserve. It averages each host's reports
|
||||
within each period, then sums hosts for fleet views. Missing reports appear as gaps.
|
||||
Historical measurements cover the selected currently registered hosts and do not
|
||||
reconstruct past readiness, port reservations, or startup slots. Reports refresh
|
||||
automatically while the page is visible, and historical queries run in a separate
|
||||
reader worker. `GET /api/capacity` supports `range=1h|24h|7d`, `host=<host ID>`, and
|
||||
`game=all|t5|t6`; host/game selections also remain in the page URL.
|
||||
|
||||
## Identity, transport and credentials
|
||||
|
||||
Enrollment tokens are random, hashed at rest, single-use and expire after 15
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { CapacityCalibration, HostReadiness } from './hosts.js';
|
||||
|
||||
export type CapacityRange = '1h' | '24h' | '7d';
|
||||
export type CapacityGame = 'all' | 't5' | 't6';
|
||||
export interface CapacityQuery { range: CapacityRange; hostId: string | null; game: CapacityGame }
|
||||
export function parseCapacityQuery(params: URLSearchParams): CapacityQuery {
|
||||
const range = params.get('range');
|
||||
const game = params.get('game');
|
||||
return { range: range === '1h' || range === '7d' ? range : '24h',
|
||||
game: game === 't5' || game === 't6' ? game : 'all', hostId: params.get('host')?.trim().slice(0, 100) || null };
|
||||
}
|
||||
|
||||
export interface CapacityHistoryPoint {
|
||||
at: number;
|
||||
cpuCoresAvailable: number | null;
|
||||
memoryBytesAvailable: number | null;
|
||||
reportingHosts: number;
|
||||
expectedHosts: number;
|
||||
}
|
||||
export interface CapacityHistory {
|
||||
range: CapacityRange;
|
||||
startAt: number;
|
||||
endAt: number;
|
||||
bucketMs: number;
|
||||
points: CapacityHistoryPoint[];
|
||||
}
|
||||
export interface CapacityHost {
|
||||
id: string;
|
||||
name: string;
|
||||
readiness: HostReadiness;
|
||||
connected: boolean;
|
||||
updatedAt: string | null;
|
||||
eligible: boolean;
|
||||
games: ('t5' | 't6')[];
|
||||
reasons: string[];
|
||||
cpuCoresAvailable: number | null;
|
||||
memoryBytesAvailable: number | null;
|
||||
portsAvailable: number;
|
||||
totalPorts: number;
|
||||
estimatedServers: number | null;
|
||||
limits: { cpu: number | null; memory: number | null; ports: number };
|
||||
limitingResource: 'CPU' | 'Memory' | 'Ports' | null;
|
||||
calibration: CapacityCalibration;
|
||||
}
|
||||
export interface CapacityCurrent {
|
||||
updatedAt: string | null;
|
||||
summary: {
|
||||
estimatedServers: number | null;
|
||||
cpuCoresAvailable: number | null;
|
||||
memoryBytesAvailable: number | null;
|
||||
portsAvailable: number | null;
|
||||
eligibleHosts: number;
|
||||
totalHosts: number;
|
||||
startupSlots: number | null;
|
||||
startupLimit: number | null;
|
||||
activeStartups: number | null;
|
||||
queued: number | null;
|
||||
};
|
||||
hosts: CapacityHost[];
|
||||
hostOptions: { id: string; name: string }[];
|
||||
warnings: string[];
|
||||
}
|
||||
export interface CapacityData { query: CapacityQuery; current: CapacityCurrent; history: CapacityHistory }
|
||||
@@ -3,7 +3,7 @@ import { fleetSettings } from '../settings.js';
|
||||
import { subscribe } from '../pipeline/bus.js';
|
||||
import { capacityCalibration } from './calibration.js';
|
||||
import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js';
|
||||
import type { Host, NativePlacement } from '../../hosts.js';
|
||||
import type { CapacityCalibration, Host, NativePlacement } from '../../hosts.js';
|
||||
import type { LaunchAvailability } from '../../deployment-plan.js';
|
||||
|
||||
interface HostAvailability {
|
||||
@@ -12,6 +12,7 @@ interface HostAvailability {
|
||||
freeMemory: number | null;
|
||||
runningCpu: number;
|
||||
runningMemory: number;
|
||||
calibration: CapacityCalibration;
|
||||
ports: { port: number; bind_address: string }[];
|
||||
}
|
||||
interface Snapshot {
|
||||
@@ -58,6 +59,7 @@ function observeHost(host: Host): HostAvailability {
|
||||
const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
|
||||
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
|
||||
return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
|
||||
calibration: { running, startup },
|
||||
ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
|
||||
}
|
||||
|
||||
@@ -90,13 +92,20 @@ subscribe(event => {
|
||||
if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
|
||||
});
|
||||
|
||||
function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
|
||||
export function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
|
||||
const bind = bindAddress ?? snapshot.host.bindAddress;
|
||||
const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
|
||||
&& (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
|
||||
return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
|
||||
}
|
||||
|
||||
/** Share the launch estimate's already observed state without rerunning placement. */
|
||||
export function readAvailabilitySnapshot(): Snapshot | null {
|
||||
const snapshot = snapshots.get(db());
|
||||
if (!snapshot) scheduleAvailabilityRefresh();
|
||||
return snapshot ?? null;
|
||||
}
|
||||
|
||||
/** Estimates only: no placement, historical scans, reservations, or live host requests. */
|
||||
export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
|
||||
const snapshot = snapshots.get(db());
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
|
||||
|
||||
const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
|
||||
'1h': { duration: 3_600_000, bucketMs: 60_000 },
|
||||
'24h': { duration: 86_400_000, bucketMs: 600_000 },
|
||||
'7d': { duration: 7 * 86_400_000, bucketMs: 3_600_000 }
|
||||
};
|
||||
|
||||
/** Read on the report worker: heartbeat history can contain millions of rows. */
|
||||
export function readCapacityHistory(query: CapacityQuery, now = Date.now()): CapacityHistory {
|
||||
const handle = db();
|
||||
ensureHostSchema(handle);
|
||||
const { duration, bucketMs } = windows[query.range];
|
||||
const startAt = now - duration;
|
||||
const count = duration / bucketMs;
|
||||
const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts
|
||||
WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`)
|
||||
.all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
|
||||
// The primary key bounds each host scan to the selected interval. Aggregate
|
||||
// before crossing the worker boundary; do not return raw heartbeat rows.
|
||||
const samples = handle.prepare(`SELECT
|
||||
MIN(?, CAST(((CAST(strftime('%s',sampled_at) AS INTEGER)*1000
|
||||
+ CAST(substr(strftime('%f',sampled_at),4,3) AS INTEGER)) - ?) / ? AS INTEGER)) AS bucket,
|
||||
AVG(MAX(0,cpu_count * (0.95 - cpu_usage_percent / 100.0))) AS cpu,
|
||||
AVG(MAX(0,memory_available - 500000000)) AS memory
|
||||
FROM ec_host_resource_samples
|
||||
WHERE host_id=? AND sampled_at>=? AND sampled_at<=?
|
||||
AND cpu_count>0 AND cpu_usage_percent BETWEEN 0 AND 100
|
||||
AND memory_total>0 AND memory_available BETWEEN 0 AND memory_total
|
||||
GROUP BY bucket`);
|
||||
const points = Array.from({ length: count }, (_, index) => ({
|
||||
at: startAt + index * bucketMs,
|
||||
cpuCoresAvailable: null as number | null,
|
||||
memoryBytesAvailable: null as number | null,
|
||||
reportingHosts: 0,
|
||||
expectedHosts: 0
|
||||
}));
|
||||
for (const host of hosts) {
|
||||
const createdAt = Date.parse(host.created_at);
|
||||
for (const point of points) {
|
||||
if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++;
|
||||
}
|
||||
const buckets = samples.all(count - 1, startAt, bucketMs, host.id,
|
||||
new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[];
|
||||
for (const bucket of buckets) {
|
||||
const point = points[bucket.bucket];
|
||||
if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue;
|
||||
// Average each host first so one host reporting more frequently
|
||||
// cannot change another host's contribution to fleet headroom.
|
||||
point.reportingHosts++;
|
||||
point.cpuCoresAvailable = (point.cpuCoresAvailable ?? 0) + bucket.cpu;
|
||||
point.memoryBytesAvailable = (point.memoryBytesAvailable ?? 0) + bucket.memory;
|
||||
}
|
||||
}
|
||||
for (const point of points) {
|
||||
// A missing host is unknown, not zero available capacity. Keep the gap
|
||||
// visible instead of presenting a partial sum as the fleet total.
|
||||
if (point.reportingHosts !== point.expectedHosts) {
|
||||
point.cpuCoresAvailable = null;
|
||||
point.memoryBytesAvailable = null;
|
||||
}
|
||||
}
|
||||
return { range: query.range, startAt, endAt: now, bucketMs, points };
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { setImmediate as yieldToController } from 'node:timers/promises';
|
||||
import { config } from '../config.js';
|
||||
import { newestReadyLibraryId } from '../game-files/index.js';
|
||||
import { HEARTBEAT_MAX_AGE_MS } from './index.js';
|
||||
import { freePorts, readAvailabilitySnapshot } from './availability.js';
|
||||
import type { CapacityCurrent, CapacityHost, CapacityQuery } from '../../capacity.js';
|
||||
|
||||
/** The page and collection preview share heartbeat/lifecycle capacity snapshots. */
|
||||
export async function readCapacityCurrent(query: CapacityQuery): Promise<CapacityCurrent> {
|
||||
let snapshot = readAvailabilitySnapshot();
|
||||
if (!snapshot) {
|
||||
// The initial snapshot is populated after the request yields, without
|
||||
// historical scans, placement, or a request to any host.
|
||||
await yieldToController();
|
||||
snapshot = readAvailabilitySnapshot();
|
||||
}
|
||||
if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'],
|
||||
summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null,
|
||||
eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } };
|
||||
const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') };
|
||||
const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game];
|
||||
const now = Date.now();
|
||||
const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => {
|
||||
const { host, calibration } = observation;
|
||||
const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
||||
const connected = host.connected && fresh;
|
||||
const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some(
|
||||
library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state)));
|
||||
const reasons: string[] = [];
|
||||
if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.');
|
||||
if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
|
||||
if (host.decommission) reasons.push('Host is being removed.');
|
||||
if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
|
||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||
if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.');
|
||||
if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
|
||||
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
|
||||
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
|
||||
const cpuCoresAvailable = fresh ? observation.freeCpu : null;
|
||||
const memoryBytesAvailable = fresh ? observation.freeMemory : null;
|
||||
const portsAvailable = freePorts(observation);
|
||||
const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
|
||||
memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)),
|
||||
ports: portsAvailable };
|
||||
const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports);
|
||||
const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null
|
||||
: limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU';
|
||||
if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.');
|
||||
else if (eligible && estimatedServers === 0) reasons.push(`No additional servers currently fit: ${limitingResource?.toLowerCase()} capacity is full.`);
|
||||
return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat,
|
||||
eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable,
|
||||
portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration };
|
||||
}).sort((first, second) => first.name.localeCompare(second.name));
|
||||
const eligible = hosts.filter(host => host.eligible);
|
||||
const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') =>
|
||||
eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
|
||||
const warnings: string[] = [];
|
||||
if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.');
|
||||
else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.');
|
||||
else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`);
|
||||
if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.');
|
||||
if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.');
|
||||
return { updatedAt: snapshot.updatedAt, hosts,
|
||||
hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'),
|
||||
memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
|
||||
eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
|
||||
startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } };
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import { capacityHistoryReport } from '../reports/index.js';
|
||||
import { readCapacityCurrent } from './capacity-overview.js';
|
||||
import type { CapacityData, CapacityQuery } from '../../capacity.js';
|
||||
|
||||
export async function capacityPage(query: CapacityQuery): Promise<CapacityData> {
|
||||
const current = await readCapacityCurrent(query);
|
||||
if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.');
|
||||
return { query, current, history: await capacityHistoryReport(query) };
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import type { MetricsQuery, MetricsData } from '../../metrics.js';
|
||||
import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js';
|
||||
import type { CapacityHistory, CapacityQuery } from '../../capacity.js';
|
||||
import { ensureHostSchema } from '../hosts/schema.js';
|
||||
|
||||
class ReportWorker {
|
||||
private worker: Worker | null = null;
|
||||
@@ -17,8 +19,9 @@ class ReportWorker {
|
||||
await worker?.terminate();
|
||||
}
|
||||
run<T>(kind: string, input: unknown): Promise<T> {
|
||||
const requestInput = input as { query?: MetricsQuery; now?: number };
|
||||
const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
|
||||
const requestInput = input as { query?: unknown; now?: number };
|
||||
const timed = kind === 'metrics' || kind === 'capacity';
|
||||
const key = JSON.stringify(timed ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
|
||||
const flight = this.flights.get(key);
|
||||
if (flight) return flight;
|
||||
if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.'));
|
||||
@@ -58,7 +61,13 @@ class ReportWorker {
|
||||
}
|
||||
const metrics = new ReportWorker();
|
||||
const leaderboard = new ReportWorker();
|
||||
const capacity = new ReportWorker();
|
||||
export const metricsReport = (query: MetricsQuery) => metrics.run<MetricsData>('metrics', { query, now: Date.now() });
|
||||
export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run<LeaderboardResponse>('leaderboard', query);
|
||||
export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options });
|
||||
export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); }
|
||||
export const capacityHistoryReport = (query: CapacityQuery) => {
|
||||
ensureHostSchema(db());
|
||||
// Game changes current placement estimates, not measured host resources.
|
||||
return capacity.run<CapacityHistory>('capacity', { query: { ...query, game: 'all' }, now: Date.now() });
|
||||
};
|
||||
export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close(), capacity.close()]); }
|
||||
|
||||
@@ -3,6 +3,7 @@ import { db } from '../db/index.js';
|
||||
import { readMetrics } from '../fleet/metrics.js';
|
||||
import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js';
|
||||
import { getPlayerProfile } from '../fleet/player-profile.js';
|
||||
import { readCapacityHistory } from '../hosts/capacity-history.js';
|
||||
|
||||
const cache = new Map<string, { value: unknown; bytes: number }>();
|
||||
let bytes = 0;
|
||||
@@ -11,12 +12,18 @@ parentPort!.on('message', ({ id, kind, input }) => {
|
||||
try {
|
||||
const result = db().transaction(() => {
|
||||
const topic = kind === 'metrics' ? 'metrics' : 'leaderboard';
|
||||
const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number };
|
||||
const now = kind === 'metrics' ? input.now : Date.now();
|
||||
const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]);
|
||||
// Capacity refreshes every 30 seconds without adding write triggers
|
||||
// to the heartbeat path. It also has its own worker, so large
|
||||
// history ranges cannot queue ahead of fleet or player reports.
|
||||
const revision = kind === 'capacity' ? 0
|
||||
: (db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number }).revision;
|
||||
const timed = kind === 'metrics' || kind === 'capacity';
|
||||
const now = timed ? input.now : Date.now();
|
||||
const key = JSON.stringify([kind, revision, timed ? Math.floor(now / 30_000) : 0, timed ? input.query : input]);
|
||||
const saved = cache.get(key);
|
||||
if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; }
|
||||
const value = kind === 'metrics' ? readMetrics(input.query, now)
|
||||
: kind === 'capacity' ? readCapacityHistory(input.query, now)
|
||||
: kind === 'leaderboard' ? getLeaderboard(input)
|
||||
: { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) };
|
||||
const size = Buffer.byteLength(JSON.stringify(value));
|
||||
|
||||
@@ -118,6 +118,7 @@
|
||||
{#if preview.availability && preview.servers.length}
|
||||
<section class="availability" aria-label="Latest reported capacity">
|
||||
<h3>Latest reported capacity</h3>
|
||||
<a href="/capacity" target="_blank" rel="noopener">View capacity and history ↗</a>
|
||||
<p class="muted">{preview.availability.estimatedAvailable === null ? 'An estimate is not available yet.' : `${preview.availability.estimatedAvailable} additional servers estimated to fit.`} {preview.availability.requested} requested.</p>
|
||||
<dl>
|
||||
<div><dt>CPU available</dt><dd>{preview.availability.cpuCoresAvailable === null ? 'Unknown' : `${preview.availability.cpuCoresAvailable.toFixed(1)} cores`}</dd></div>
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
<script lang="ts">
|
||||
import type { CapacityHistory, CapacityHistoryPoint } from '$lib/capacity.js';
|
||||
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
|
||||
|
||||
let { history, scope }: { history: CapacityHistory; scope: string } = $props();
|
||||
let metric = $state<'cpu' | 'memory'>('cpu');
|
||||
let selectedAt = $state<number | null>(null);
|
||||
let tableOpen = $state(false);
|
||||
const points = $derived(history.points);
|
||||
const label = $derived(metric === 'cpu' ? 'CPU cores' : 'Memory (GiB)');
|
||||
const selectedIndex = $derived.by(() => {
|
||||
if (selectedAt !== null && points.length) {
|
||||
let closest = 0;
|
||||
for (let index = 1; index < points.length; index++) {
|
||||
if (Math.abs(points[index].at - selectedAt) < Math.abs(points[closest].at - selectedAt)) closest = index;
|
||||
}
|
||||
return closest;
|
||||
}
|
||||
for (let index = points.length - 1; index >= 0; index--) if (value(points[index]) !== null) return index;
|
||||
return Math.max(0, points.length - 1);
|
||||
});
|
||||
const selected = $derived(points[selectedIndex]);
|
||||
const measured = $derived(points.filter(point => value(point) !== null));
|
||||
const ceiling = $derived.by(() => {
|
||||
const max = Math.max(1, ...measured.map(point => value(point)!));
|
||||
const step = 10 ** Math.floor(Math.log10(max)) / 2;
|
||||
return Math.ceil(max / step) * step;
|
||||
});
|
||||
const segments = $derived.by(() => {
|
||||
const result: { x: number; y: number }[][] = [];
|
||||
let segment: { x: number; y: number }[] = [];
|
||||
for (const point of points) {
|
||||
const amount = value(point);
|
||||
if (amount === null) { if (segment.length) result.push(segment); segment = []; }
|
||||
else segment.push({ x: x(point.at), y: y(amount) });
|
||||
}
|
||||
if (segment.length) result.push(segment);
|
||||
return result;
|
||||
});
|
||||
const ticks = [1, .75, .5, .25, 0];
|
||||
const numbers = new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 });
|
||||
function value(point: CapacityHistoryPoint): number | null {
|
||||
return metric === 'cpu' ? point.cpuCoresAvailable : point.memoryBytesAvailable === null ? null : point.memoryBytesAvailable / 1024 ** 3;
|
||||
}
|
||||
function x(at: number) { return Math.max(0, Math.min(1000, (at - history.startAt) / Math.max(1, history.endAt - history.startAt) * 1000)); }
|
||||
function y(amount: number) { return 230 - Math.max(0, Math.min(1, amount / ceiling)) * 220; }
|
||||
function amount(value: number | null, unit: string) { return value === null ? 'No complete report' : `${numbers.format(value)} ${unit}`; }
|
||||
function describe(point: CapacityHistoryPoint) {
|
||||
return `${$timeFormat.period(point.at)}. ${amount(value(point), metric === 'cpu' ? 'cores available' : 'GiB available')}. ${point.reportingHosts} of ${point.expectedHosts} hosts reporting.`;
|
||||
}
|
||||
function inspect(event: PointerEvent) {
|
||||
if (!points.length || (event.type === 'pointermove' && event.pointerType === 'touch')) return;
|
||||
const rect = (event.currentTarget as HTMLElement).getBoundingClientRect();
|
||||
const at = history.startAt + Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)) * (history.endAt - history.startAt);
|
||||
let closest = points[0];
|
||||
for (const point of points) if (Math.abs(point.at - at) < Math.abs(closest.at - at)) closest = point;
|
||||
selectedAt = closest.at;
|
||||
}
|
||||
function move(event: KeyboardEvent) {
|
||||
if (!points.length || !['ArrowLeft', 'ArrowRight', 'Home', 'End'].includes(event.key)) return;
|
||||
event.preventDefault();
|
||||
const index = event.key === 'Home' ? 0 : event.key === 'End' ? points.length - 1
|
||||
: Math.max(0, Math.min(points.length - 1, selectedIndex + (event.key === 'ArrowLeft' ? -1 : 1)));
|
||||
selectedAt = points[index].at;
|
||||
}
|
||||
</script>
|
||||
|
||||
<section class="panel" aria-labelledby="capacity-history-title">
|
||||
<div class="heading">
|
||||
<div><h2 id="capacity-history-title">Capacity over time</h2><p>Measured resource headroom · {scope}</p></div>
|
||||
<div class="mode" role="group" aria-label="Chart resource">
|
||||
<button type="button" aria-pressed={metric === 'cpu'} onclick={() => metric = 'cpu'}>CPU</button>
|
||||
<button type="button" aria-pressed={metric === 'memory'} onclick={() => metric = 'memory'}>Memory</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="axis-title">{label} available</div>
|
||||
<div class="chart-grid">
|
||||
<div class="y-axis" aria-hidden="true">{#each ticks as tick}<span style:top={`${(1 - tick) * 100}%`}>{numbers.format(ceiling * tick)}</span>{/each}</div>
|
||||
<div class="plot" role="slider" tabindex="0" aria-label={`${label} history. Use left and right arrows to inspect a period.`}
|
||||
aria-valuemin="0" aria-valuemax={Math.max(0, points.length - 1)} aria-valuenow={selectedIndex}
|
||||
aria-valuetext={selected ? describe(selected) : 'No history available yet'}
|
||||
onpointermove={inspect} onpointerdown={inspect} onkeydown={move}>
|
||||
<svg viewBox="0 0 1000 240" preserveAspectRatio="none" aria-hidden="true">
|
||||
{#each ticks as tick}<line class="grid-line" x1="0" x2="1000" y1={y(ceiling * tick)} y2={y(ceiling * tick)} />{/each}
|
||||
{#each segments as segment}
|
||||
{#if segment.length > 1}<path class="area" d={`M${segment[0].x},230 ${segment.map(point => `L${point.x},${point.y}`).join(' ')} L${segment[segment.length - 1].x},230 Z`} />{/if}
|
||||
<path class="trend" d={segment.map((point, index) => `${index ? 'L' : 'M'}${point.x},${point.y}`).join(' ')} />
|
||||
{#if segment.length === 1}<circle class="point" cx={segment[0].x} cy={segment[0].y} r="3" />{/if}
|
||||
{/each}
|
||||
{#if selected}<line class="cursor" x1={x(selected.at)} x2={x(selected.at)} y1="5" y2="230" />
|
||||
{#if value(selected) !== null}<circle class="selected-point" cx={x(selected.at)} cy={y(value(selected)!)} r="4" />{/if}
|
||||
{/if}
|
||||
</svg>
|
||||
{#if !measured.length}<div class="empty-chart"><strong>Waiting for capacity history</strong><span>Fresh host reports build this timeline automatically.</span></div>{/if}
|
||||
</div>
|
||||
<div class="x-axis" aria-hidden="true"><span>{$timeFormat.period(history.startAt)}</span><span>{$timeFormat.period(history.endAt)}</span></div>
|
||||
</div>
|
||||
{#if selected}
|
||||
<div class="readout" aria-label="Selected history period">
|
||||
<div class="selected-time"><span>Selected period</span><strong>{$timeFormat.period(selected.at)}</strong></div>
|
||||
<dl><div><dt>CPU available</dt><dd>{amount(selected.cpuCoresAvailable, 'cores')}</dd></div>
|
||||
<div><dt>Memory available</dt><dd>{amount(selected.memoryBytesAvailable === null ? null : selected.memoryBytesAvailable / 1024 ** 3, 'GiB')}</dd></div>
|
||||
<div><dt>Reporting hosts</dt><dd>{selected.reportingHosts} / {selected.expectedHosts}</dd></div></dl>
|
||||
</div>
|
||||
{/if}
|
||||
<p class="note">Host headroom after the operating-system reserve, averaged per period. Gaps mean reporting is incomplete. The game filter changes current launch estimates; this history includes all workloads on the selected hosts.</p>
|
||||
<details bind:open={tableOpen}><summary>View history data · {$timeZone.replaceAll('_', ' ')}</summary>
|
||||
{#if tableOpen}
|
||||
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Scrollable data must be reachable by keyboard.) -->
|
||||
<div class="table-wrap" role="region" aria-label="Capacity history data" tabindex="0"><table>
|
||||
<caption>Measured headroom; a dash means a complete measurement is unavailable.</caption>
|
||||
<thead><tr><th scope="col">Period</th><th scope="col">CPU cores</th><th scope="col">Memory (GiB)</th><th scope="col">Reporting hosts</th></tr></thead>
|
||||
<tbody>{#each points as point (point.at)}<tr><th scope="row">{$timeFormat.period(point.at)}</th><td>{point.cpuCoresAvailable === null ? '—' : numbers.format(point.cpuCoresAvailable)}</td><td>{point.memoryBytesAvailable === null ? '—' : numbers.format(point.memoryBytesAvailable / 1024 ** 3)}</td><td>{point.reportingHosts} / {point.expectedHosts}</td></tr>{/each}</tbody>
|
||||
</table></div>
|
||||
{/if}
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.panel { background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); padding: var(--space-6); min-width: 0; }
|
||||
h2 { margin: 0; font-size: var(--text-lg); } .heading { display: flex; justify-content: space-between; align-items: flex-start; gap: var(--space-4); flex-wrap: wrap; }
|
||||
.heading p { margin: var(--space-2) 0 0; color: var(--text-secondary); font-size: var(--text-sm); }
|
||||
.mode { display: flex; gap: 3px; padding: 3px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); }
|
||||
.mode button { padding: 7px 14px; border: 0; border-radius: var(--radius-sm); background: transparent; color: var(--text-secondary); font: inherit; font-size: var(--text-xs); cursor: pointer; }
|
||||
.mode button[aria-pressed='true'] { color: var(--text-primary); background: var(--surface-raised); }
|
||||
.mode button:focus-visible, .plot:focus-visible, .table-wrap:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: 3px; }
|
||||
.axis-title { margin-top: var(--space-6); color: var(--text-tertiary); font-size: 11px; }
|
||||
.chart-grid { display: grid; grid-template-columns: 42px minmax(0, 1fr); gap: 10px; margin-top: 14px; }
|
||||
.y-axis { position: relative; height: 220px; margin-top: 10px; color: var(--text-tertiary); font-size: 11px; }
|
||||
.y-axis span { position: absolute; right: 0; transform: translateY(-50%); }
|
||||
.plot { position: relative; height: 240px; border-radius: 3px; cursor: crosshair; touch-action: pan-y; }
|
||||
svg { display: block; width: 100%; height: 100%; overflow: visible; }
|
||||
svg path, svg line { vector-effect: non-scaling-stroke; }
|
||||
.grid-line { stroke: var(--border-subtle); stroke-width: 1; } .area { fill: color-mix(in oklab, var(--accent-primary) 10%, transparent); }
|
||||
.trend { fill: none; stroke: var(--accent-primary); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; }
|
||||
.point, .selected-point { fill: var(--accent-primary-hover); } .selected-point { stroke: var(--surface); stroke-width: 2; }
|
||||
.cursor { stroke: var(--text-tertiary); stroke-width: 1; stroke-dasharray: 4 5; }
|
||||
.x-axis { grid-column: 2; display: flex; justify-content: space-between; gap: var(--space-4); color: var(--text-tertiary); font-size: 11px; }
|
||||
.x-axis span:last-child { text-align: right; }
|
||||
.empty-chart { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; flex-direction: column; gap: 10px; text-align: center; padding: var(--space-4); pointer-events: none; }
|
||||
.empty-chart strong { font-size: var(--text-sm); font-weight: var(--weight-medium); } .empty-chart span { color: var(--text-tertiary); font-size: var(--text-xs); }
|
||||
.readout { display: flex; flex-wrap: wrap; gap: var(--space-6); align-items: center; background: var(--surface-sunken); padding: var(--space-4); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); margin-top: var(--space-5); }
|
||||
.selected-time { display: grid; gap: 6px; flex: 1 1 180px; } .selected-time span, dt { color: var(--text-tertiary); font-size: 11px; } .selected-time strong { font-size: var(--text-xs); font-weight: var(--weight-medium); }
|
||||
dl { display: flex; flex-wrap: wrap; gap: var(--space-6); margin: 0; } dd { margin: 6px 0 0; font-size: var(--text-sm); font-variant-numeric: tabular-nums; }
|
||||
.note { color: var(--text-tertiary); font-size: var(--text-xs); line-height: 1.65; margin: var(--space-4) 0; max-width: 115ch; }
|
||||
summary { cursor: pointer; color: var(--text-secondary); font-size: var(--text-xs); padding: 4px 0; }
|
||||
.table-wrap { overflow: auto; max-height: 400px; margin-top: var(--space-3); } table { width: 100%; border-collapse: collapse; font-size: var(--text-xs); }
|
||||
caption { text-align: left; color: var(--text-tertiary); margin-bottom: var(--space-3); } th, td { text-align: right; padding: 12px; white-space: nowrap; border-bottom: 1px solid var(--border-subtle); font-variant-numeric: tabular-nums; }
|
||||
th:first-child { text-align: left; } tbody th { font-weight: var(--weight-regular); } thead th { color: var(--text-secondary); }
|
||||
@media (max-width: 640px) { .panel { padding: var(--space-4); } .chart-grid { grid-template-columns: 32px minmax(0, 1fr); gap: 8px; } .readout { gap: var(--space-4); } dl { gap: var(--space-4); } .x-axis { font-size: 10px; } }
|
||||
</style>
|
||||
@@ -131,6 +131,7 @@
|
||||
const tabs = [
|
||||
{ href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') },
|
||||
{ href: '/hosts', label: 'Hosts', match: (p: string) => /^\/hosts(\/|$)/.test(p) },
|
||||
{ href: '/capacity', label: 'Capacity', match: (p: string) => p === '/capacity' },
|
||||
{ href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins|game-files)(\/|$)/.test(p) },
|
||||
{ href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|maintenance|watchdog|server-keys)(\/|$)/.test(p) },
|
||||
{ href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') },
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { parseCapacityQuery } from '$lib/capacity.js';
|
||||
import { capacityPage } from '$lib/server/hosts/capacity-page.js';
|
||||
|
||||
export async function GET({ url }: { url: URL }) {
|
||||
return json(await capacityPage(parseCapacityQuery(url.searchParams)), { headers: { 'cache-control': 'no-store' } });
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { parseCapacityQuery } from '$lib/capacity.js';
|
||||
import { capacityPage } from '$lib/server/hosts/capacity-page.js';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = ({ url }) => capacityPage(parseCapacityQuery(url.searchParams));
|
||||
@@ -0,0 +1,137 @@
|
||||
<script lang="ts">
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { beforeNavigate, afterNavigate, goto } from '$app/navigation';
|
||||
import { page } from '$app/state';
|
||||
import type { CapacityData, CapacityGame, CapacityQuery, CapacityRange } from '$lib/capacity.js';
|
||||
import { hostReadinessLabel } from '$lib/hosts.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import Field from '$ui/components/primitives/Field.svelte';
|
||||
import Select from '$ui/components/primitives/Select.svelte';
|
||||
import CapacityChart from '$ui/components/capacity/CapacityChart.svelte';
|
||||
|
||||
let { data }: { data: CapacityData } = $props();
|
||||
let snapshot = $state<CapacityData | null>(null);
|
||||
let refreshing = $state(false);
|
||||
let navigating = $state(false);
|
||||
let refreshError = $state('');
|
||||
let navigationError = $state('');
|
||||
let request: AbortController | null = null;
|
||||
let sequence = 0;
|
||||
const report = $derived(snapshot ?? data);
|
||||
const summary = $derived(report.current.summary);
|
||||
const scopeName = $derived(report.query.hostId ? report.current.hostOptions.find(host => host.id === report.query.hostId)?.name ?? 'Selected host' : 'All hosts');
|
||||
const hostOptions = $derived([{ value: '', label: 'All hosts' }, ...report.current.hostOptions.map(host => ({ value: host.id, label: host.name })),
|
||||
...(report.query.hostId && !report.current.hostOptions.some(host => host.id === report.query.hostId) ? [{ value: report.query.hostId, label: 'Unavailable host' }] : [])]);
|
||||
const gameOptions = [{ value: 'all', label: 'All games' }, { value: 't5', label: 'Black Ops I (T5)' }, { value: 't6', label: 'Black Ops II (T6)' }];
|
||||
const ranges: { value: CapacityRange; label: string }[] = [{ value: '1h', label: 'Last hour' }, { value: '24h', label: '24 hours' }, { value: '7d', label: '7 days' }];
|
||||
const numbers = new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 });
|
||||
function count(value: number | null) { return value === null ? '—' : numbers.format(value); }
|
||||
function memory(value: number | null) { return value === null ? '—' : numbers.format(value / 1024 ** 3); }
|
||||
function queryParams(query: CapacityQuery) {
|
||||
const params = new URLSearchParams({ range: query.range, game: query.game });
|
||||
if (query.hostId) params.set('host', query.hostId);
|
||||
return params;
|
||||
}
|
||||
function scopeUrl(change: Partial<CapacityQuery>) { return `/capacity?${queryParams({ ...report.query, ...change })}`; }
|
||||
function cancel() { sequence++; request?.abort(); request = null; refreshing = false; }
|
||||
async function refresh(signal?: AbortSignal) {
|
||||
if (navigating || refreshing) return;
|
||||
const current = ++sequence;
|
||||
request = new AbortController();
|
||||
refreshing = true;
|
||||
try {
|
||||
const response = await fetch(`/api/capacity?${queryParams(data.query)}`, {
|
||||
signal: AbortSignal.any([request.signal, AbortSignal.timeout(25_000), ...(signal ? [signal] : [])])
|
||||
});
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not refresh capacity.');
|
||||
if (current === sequence) { snapshot = body; refreshError = ''; }
|
||||
} catch (cause) {
|
||||
if (current === sequence && !signal?.aborted && !request?.signal.aborted) refreshError = (cause as Error).message;
|
||||
return false;
|
||||
} finally { if (current === sequence) { refreshing = false; request = null; } }
|
||||
}
|
||||
async function changeScope(change: Partial<CapacityQuery>) {
|
||||
navigationError = '';
|
||||
try { await goto(scopeUrl(change), { keepFocus: true, noScroll: true }); }
|
||||
catch (cause) { navigating = false; navigationError = (cause as Error).message || 'Could not load that capacity view.'; }
|
||||
}
|
||||
beforeNavigate(() => { cancel(); navigating = true; });
|
||||
afterNavigate(() => { navigating = false; });
|
||||
$effect(() => { void data; untrack(() => { cancel(); snapshot = null; refreshError = ''; }); });
|
||||
onMount(() => { const stop = pollVisible(refresh, 30_000, false); return () => { stop(); cancel(); }; });
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Capacity · Electric Cherry</title><meta name="description" content="Available game-server capacity, resource history, and host limits." /></svelte:head>
|
||||
|
||||
<div class="page" aria-busy={navigating}>
|
||||
<SectionHeader title="Capacity" subtitle="See what your hosts can run, what is limiting them, and how headroom changes.">
|
||||
{#snippet actions()}<Button href="/hosts">Manage hosts</Button><Button disabled={refreshing || navigating} onclick={() => void refresh()}>{refreshing ? 'Refreshing…' : 'Refresh'}</Button>{/snippet}
|
||||
</SectionHeader>
|
||||
{#if refreshError || navigationError}<div class="refresh-error" role="alert"><strong>{navigationError || refreshError}</strong><span>Showing the last successfully loaded capacity report.</span></div>{/if}
|
||||
<div class="toolbar">
|
||||
<div class="filters"><Field label="Host"><Select value={report.query.hostId ?? ''} options={hostOptions} disabled={navigating} onchange={event => void changeScope({ hostId: (event.currentTarget as HTMLSelectElement).value || null })} /></Field>
|
||||
<Field label="Launch game"><Select value={report.query.game} options={gameOptions} disabled={navigating} onchange={event => void changeScope({ game: (event.currentTarget as HTMLSelectElement).value as CapacityGame })} /></Field></div>
|
||||
<nav class="ranges" aria-label="History range">{#each ranges as range}<a href={scopeUrl({ range: range.value })} aria-current={report.query.range === range.value ? 'page' : undefined} data-sveltekit-keepfocus data-sveltekit-noscroll>{range.label}</a>{/each}</nav>
|
||||
</div>
|
||||
<section class="overview" aria-labelledby="capacity-now-title">
|
||||
<div class="estimate"><div class="eyebrow" id="capacity-now-title">Available now · {scopeName}</div><div class="estimate-number">{count(summary.estimatedServers)}<span>additional servers</span></div>
|
||||
<p>{summary.estimatedServers === null ? 'Waiting for enough host information to estimate capacity.' : `Estimated to fit across ${summary.eligibleHosts} eligible ${summary.eligibleHosts === 1 ? 'host' : 'hosts'}.`} {report.query.game === 'all' ? 'Hosts ready for either T5 or T6 are counted once.' : `For ${report.query.game === 't5' ? 'Black Ops I' : 'Black Ops II'} launches.`}</p></div>
|
||||
<div class="queue-context"><a href="/provisioning"><strong>{count(summary.queued)}</strong><span>servers queued <span aria-hidden="true">→</span></span></a><p>Across the fleet. Launches wait for capacity and startup slots automatically.</p></div>
|
||||
</section>
|
||||
<div class="resources" aria-label="Current resource headroom">
|
||||
<article><span class="resource-label">CPU available</span><div class="resource-value">{count(summary.cpuCoresAvailable)}<small>cores</small></div><p>After OS reserve and pending starts</p></article>
|
||||
<article><span class="resource-label">Memory available</span><div class="resource-value">{memory(summary.memoryBytesAvailable)}<small>GiB</small></div><p>After OS reserve and pending starts</p></article>
|
||||
<article><span class="resource-label">Ports available</span><div class="resource-value">{count(summary.portsAvailable)}<small>ports</small></div><p>Free game ports on eligible hosts</p></article>
|
||||
<article><span class="resource-label">Startup slots</span><div class="resource-value">{count(summary.startupSlots)}<small>{summary.startupLimit === null ? 'available' : `/ ${summary.startupLimit}`}</small></div><p>{count(summary.activeStartups)} starting · controller-wide</p></article>
|
||||
</div>
|
||||
<div class="freshness"><span>{report.current.updatedAt ? `Capacity updated ${$timeFormat.dateTime(report.current.updatedAt)}` : 'Waiting for host reports'}</span><span>Refreshes every 30 seconds while visible</span></div>
|
||||
{#if report.current.warnings.length}<aside class="warnings" aria-label="Capacity notices">{#each report.current.warnings as warning}<p>{warning}</p>{/each}</aside>{/if}
|
||||
<CapacityChart history={report.history} scope={scopeName} />
|
||||
<section class="hosts" aria-labelledby="host-capacity-title">
|
||||
<div class="section-heading"><div><h2 id="host-capacity-title">Host capacity</h2><p>Find the resource that limits the next launch.</p></div><span>{summary.eligibleHosts} / {summary.totalHosts} eligible</span></div>
|
||||
{#if !report.current.hosts.length}<div class="empty-hosts"><h3>{report.query.hostId ? 'This host is unavailable' : 'No hosts yet'}</h3><p>{report.query.hostId ? 'Choose another host to view its capacity.' : 'Connect a host to start seeing capacity and history.'}</p><Button href="/hosts">Open hosts</Button></div>
|
||||
{:else}<div class="host-grid">{#each report.current.hosts as host (host.id)}<article class="host-card">
|
||||
<div class="host-title"><h3><a href={`/hosts/${host.id}`}>{host.name} <span aria-hidden="true">↗</span></a></h3><span class="status" class:eligible={host.eligible}>{!host.connected ? 'Offline' : hostReadinessLabel(host.readiness)}</span></div>
|
||||
<div class="host-estimate"><strong>{count(host.estimatedServers)}</strong><span>additional servers{#if host.eligible && host.limitingResource}<small>Limited by {host.limitingResource.toLowerCase()}</small>{/if}</span></div>
|
||||
<div class="game-tags">{#if host.games.length}{#each host.games as game}<span>{game === 't5' ? 'Black Ops I' : 'Black Ops II'}</span>{/each}{:else}<span>No games ready</span>{/if}</div>
|
||||
<dl class="host-resources"><div><dt>CPU</dt><dd>{count(host.cpuCoresAvailable)} <small>cores</small></dd><span>{count(host.limits.cpu)} servers fit</span></div><div><dt>Memory</dt><dd>{memory(host.memoryBytesAvailable)} <small>GiB</small></dd><span>{count(host.limits.memory)} servers fit</span></div><div><dt>Ports</dt><dd>{host.portsAvailable} <small>/ {host.totalPorts}</small></dd><span>{host.limits.ports} servers fit</span></div></dl>
|
||||
{#if host.reasons.length}<ul class="host-reasons">{#each host.reasons as reason}<li>{reason}</li>{/each}</ul>{/if}
|
||||
<details class="allowances"><summary>Per-server resource estimates</summary>
|
||||
<div class="allowance-grid">{#each [{ label: 'Running', value: host.calibration.running }, { label: 'Starting', value: host.calibration.startup }] as allowance}<div><strong>{allowance.label}</strong><p>{count(allowance.value.cpuCores)} cores · {memory(allowance.value.memoryBytes)} GiB</p><span>{allowance.value.source === 'bootstrap' ? 'Default allowance' : `${allowance.value.source === 'host' ? 'Host' : 'Fleet'} observations · ${allowance.value.samples} samples`}</span></div>{/each}</div>
|
||||
<p>Starts may need more headroom than running servers. Provisioning checks actual capacity before reserving a host and port.</p>
|
||||
</details>
|
||||
<p class="host-updated">{host.updatedAt ? `Reported ${$timeFormat.dateTime(host.updatedAt)}` : 'No host report yet'}</p>
|
||||
</article>{/each}</div>{/if}
|
||||
</section>
|
||||
<details class="methodology"><summary>How capacity is estimated</summary><p>Additional servers are estimated from each eligible host’s available CPU, memory, and unreserved ports, using its learned running-server allowance. The smallest resource allowance sets that host’s limit. Shared hosts are counted once.</p><p>This is a planning estimate. Queued launches do not reserve capacity, and starting servers may require a larger allowance. Actual placement also checks current host readiness, game files, runtime, and available startup slots.</p><p>The timeline shows measured host resource headroom after the operating-system reserve. It does not reconstruct historical launch eligibility, queued jobs, or port availability.</p></details>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page { display: grid; gap: var(--space-5); min-width: 0; } .toolbar { display: flex; align-items: end; justify-content: space-between; flex-wrap: wrap; gap: var(--space-4); }
|
||||
.filters { display: grid; grid-template-columns: minmax(180px, 260px) minmax(180px, 260px); gap: var(--space-3); }
|
||||
.ranges { display: flex; gap: 3px; padding: 4px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); }
|
||||
.ranges a { padding: 9px 13px; border-radius: var(--radius-sm); text-decoration: none; font-size: var(--text-xs); color: var(--text-secondary); white-space: nowrap; }
|
||||
.ranges a[aria-current] { color: var(--text-primary); background: var(--surface-raised); } .ranges a:focus-visible { outline: 2px solid var(--accent-primary); outline-offset: 2px; }
|
||||
.overview { display: flex; justify-content: space-between; align-items: center; gap: var(--space-8); background: var(--surface); border: 1px solid var(--border-subtle); border-left: 3px solid var(--accent-primary); border-radius: var(--radius-lg); padding: var(--space-6) var(--space-8); }
|
||||
.eyebrow { font-size: var(--text-xs); color: var(--text-secondary); font-weight: var(--weight-medium); } .estimate-number { display: flex; flex-wrap: wrap; align-items: baseline; column-gap: var(--space-4); margin-top: var(--space-3); font-size: 56px; font-weight: var(--weight-semibold); line-height: 1.1; font-variant-numeric: tabular-nums; letter-spacing: -.035em; }
|
||||
.estimate-number span { font-size: var(--text-md); font-weight: var(--weight-regular); color: var(--text-secondary); letter-spacing: normal; } .estimate p, .queue-context p { color: var(--text-tertiary); font-size: var(--text-xs); line-height: 1.6; margin: var(--space-3) 0 0; }
|
||||
.queue-context { flex: 0 0 210px; border-left: 1px solid var(--border-strong); padding-left: var(--space-6); } .queue-context a { color: var(--text-primary); text-decoration: none; display: flex; align-items: baseline; gap: var(--space-3); } .queue-context a:hover span { color: var(--accent-primary-hover); } .queue-context strong { font-size: var(--text-2xl); font-weight: var(--weight-medium); font-variant-numeric: tabular-nums; } .queue-context span { color: var(--text-secondary); font-size: var(--text-xs); }
|
||||
.resources { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: var(--space-3); } .resources article { background: var(--surface); padding: var(--space-5); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); }
|
||||
.resource-label { color: var(--text-secondary); font-size: var(--text-xs); } .resource-value { font-size: var(--text-2xl); font-weight: var(--weight-medium); margin-top: var(--space-3); font-variant-numeric: tabular-nums; display: flex; align-items: baseline; flex-wrap: wrap; gap: 7px; } .resource-value small { font-size: var(--text-sm); color: var(--text-tertiary); font-weight: var(--weight-regular); } .resources p { color: var(--text-tertiary); font-size: 11px; line-height: 1.5; margin: var(--space-2) 0 0; }
|
||||
.freshness { display: flex; justify-content: space-between; flex-wrap: wrap; gap: var(--space-2); color: var(--text-tertiary); font-size: 11px; margin-top: -6px; }
|
||||
.warnings, .refresh-error { border-left: 2px solid var(--state-warning); padding: var(--space-3) var(--space-4); background: var(--surface); border-radius: 0 var(--radius-sm) var(--radius-sm) 0; font-size: var(--text-xs); color: var(--text-secondary); line-height: 1.6; } .warnings p { margin: 0; } .warnings p + p { margin-top: var(--space-2); } .refresh-error { display: grid; gap: var(--space-1); }
|
||||
.hosts { display: grid; gap: var(--space-4); } .section-heading { display: flex; justify-content: space-between; align-items: center; gap: var(--space-4); } h2 { font-size: var(--text-lg); margin: 0; } .section-heading p { font-size: var(--text-sm); color: var(--text-tertiary); margin: 6px 0 0; } .section-heading > span { color: var(--text-secondary); font-size: var(--text-xs); white-space: nowrap; }
|
||||
.host-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: var(--space-4); } .host-card, .empty-hosts { background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); padding: var(--space-5); min-width: 0; } .host-title { display: flex; justify-content: space-between; gap: var(--space-3); align-items: center; } h3 { font-size: var(--text-md); margin: 0; } .host-title h3 { overflow-wrap: anywhere; } .host-title a { text-decoration: none; color: var(--text-primary); } .host-title a:hover { color: var(--accent-primary-hover); } .host-title a span { color: var(--text-tertiary); font-size: var(--text-sm); }
|
||||
.status { padding: 3px 8px; font-size: 11px; background: var(--surface-raised); color: var(--text-secondary); border: 1px solid var(--border-subtle); border-radius: var(--radius-pill); white-space: nowrap; } .status.eligible { color: var(--state-online); }
|
||||
.host-estimate { display: flex; align-items: center; gap: var(--space-3); margin-top: var(--space-5); } .host-estimate strong { font-size: var(--text-2xl); font-weight: var(--weight-medium); font-variant-numeric: tabular-nums; } .host-estimate span { font-size: var(--text-xs); color: var(--text-secondary); } .host-estimate small { display: block; color: var(--text-tertiary); margin-top: 4px; font-size: 11px; }
|
||||
.game-tags { display: flex; flex-wrap: wrap; gap: var(--space-2); margin-top: var(--space-3); } .game-tags span { font-size: 10px; color: var(--text-secondary); background: var(--surface-sunken); padding: 4px 7px; border-radius: 4px; }
|
||||
.host-resources { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: var(--space-3); margin: var(--space-5) 0 0; padding: var(--space-4) 0; border-top: 1px solid var(--border-subtle); border-bottom: 1px solid var(--border-subtle); } .host-resources dt { font-size: 11px; color: var(--text-tertiary); } .host-resources dd { margin: 8px 0 5px; font-size: var(--text-md); font-variant-numeric: tabular-nums; } .host-resources dd small, .host-resources div > span { font-size: 11px; color: var(--text-tertiary); }
|
||||
.host-reasons { padding-left: 16px; font-size: var(--text-xs); color: var(--text-secondary); line-height: 1.7; margin: var(--space-3) 0 0; } .allowances { margin-top: var(--space-4); } summary { cursor: pointer; font-size: var(--text-xs); color: var(--text-secondary); padding: 4px 0; } .allowance-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--space-3); background: var(--surface-sunken); border-radius: var(--radius-sm); padding: var(--space-3); margin-top: var(--space-3); } .allowance-grid strong { font-size: var(--text-xs); font-weight: var(--weight-medium); } .allowance-grid p { font-size: var(--text-xs); margin: 8px 0 4px; color: var(--text-secondary); } .allowance-grid span { font-size: 10px; color: var(--text-tertiary); } .allowances > p, .host-updated { font-size: 11px; line-height: 1.6; color: var(--text-tertiary); margin: var(--space-3) 0 0; }
|
||||
.methodology { padding: 0 2px; } .methodology p { font-size: var(--text-xs); color: var(--text-tertiary); line-height: 1.7; max-width: 115ch; } .empty-hosts p { color: var(--text-secondary); font-size: var(--text-sm); line-height: 1.6; }
|
||||
@media (max-width: 900px) { .resources { grid-template-columns: repeat(2, minmax(0, 1fr)); } .host-grid { grid-template-columns: 1fr; } }
|
||||
@media (max-width: 600px) { .page { gap: var(--space-4); } .filters { grid-template-columns: 1fr 1fr; width: 100%; } .overview { padding: var(--space-5); flex-direction: column; align-items: stretch; gap: var(--space-4); } .estimate-number { font-size: 48px; } .queue-context { flex-basis: auto; border-left: 0; border-top: 1px solid var(--border-subtle); padding: var(--space-3) 0 0; } .queue-context p { margin-top: 4px; } .queue-context strong { font-size: var(--text-xl); } .resources article { padding: var(--space-4); } .resource-value { font-size: 28px; } .host-card { padding: var(--space-4); } .section-heading { align-items: flex-start; } .ranges { width: 100%; } .ranges a { text-align: center; flex: 1; } }
|
||||
</style>
|
||||
@@ -39,7 +39,9 @@
|
||||
|
||||
<svelte:head><title>Hosts · Electric Cherry</title></svelte:head>
|
||||
<section class="page">
|
||||
<SectionHeader title="Hosts" subtitle="Connect game-server hosts. Runtime and game files are prepared automatically." />
|
||||
<SectionHeader title="Hosts" subtitle="Connect game-server hosts. Runtime and game files are prepared automatically.">
|
||||
{#snippet actions()}<Button href="/capacity">View capacity</Button>{/snippet}
|
||||
</SectionHeader>
|
||||
{#if error}<p role="alert" class="ui-error">{error}</p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if canManage}<section class="panel">
|
||||
|
||||
@@ -7,7 +7,7 @@ const allows = (role: UserRole, route: string | null, method = 'GET') => roleAll
|
||||
|
||||
describe('management route permissions', () => {
|
||||
it.each([
|
||||
'/', '/api/fleet', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/', '/api/fleet', '/capacity', '/api/capacity', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/api/metrics', '/metrics', '/api/matches', '/matches/[id]', '/api/leaderboard',
|
||||
'/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]',
|
||||
'/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events',
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import type { CapacityQuery } from '../../src/lib/capacity.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const epoch = Date.UTC(2026, 9, 5, 12);
|
||||
const GiB = 1024 ** 3;
|
||||
const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' };
|
||||
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
|
||||
|
||||
async function host(id: string, createdAt = epoch - 7 * 86_400_000) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
||||
ensureHostSchema(db());
|
||||
db().prepare('INSERT INTO ec_hosts (id,name,created_at) VALUES (?,?,?)').run(id, id, new Date(createdAt).toISOString());
|
||||
}
|
||||
|
||||
async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores = 8) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO ec_host_resource_samples
|
||||
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES (?,?,?,?,?,?)`)
|
||||
.run(id, new Date(at).toISOString(), cores, cpu, 16 * GiB, memory);
|
||||
}
|
||||
|
||||
describe('capacity history', () => {
|
||||
it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
|
||||
await host('one'); await host('two');
|
||||
await sample('one', epoch - 55_000, 0, 8 * GiB);
|
||||
await sample('one', epoch - 30_000, 50, 4 * GiB);
|
||||
await sample('two', epoch - 45_000, 50, 2 * GiB, 4);
|
||||
await sample('one', epoch - 90_000);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory(query, epoch);
|
||||
expect(history.points).toHaveLength(60);
|
||||
expect(history.points.at(-1)).toMatchObject({ at: epoch - 60_000,
|
||||
memoryBytesAvailable: 8 * GiB - 1_000_000_000,
|
||||
reportingHosts: 2, expectedHosts: 2 });
|
||||
expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(7.4);
|
||||
expect(history.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 2,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
expect(history.points[0]).toMatchObject({ reportingHosts: 0, expectedHosts: 2,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
});
|
||||
|
||||
it('scopes history to the selected registered host and does not add hosts before they existed', async () => {
|
||||
await host('one'); await host('two', epoch - 30_000); await host('removed');
|
||||
await sample('one', epoch - 90_000);
|
||||
await sample('one', epoch - 15_000);
|
||||
await sample('two', epoch - 15_000);
|
||||
await sample('removed', epoch - 15_000);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?').run(new Date(epoch).toISOString(), 'removed');
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const fleet = readCapacityHistory(query, epoch);
|
||||
expect(fleet.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 1 });
|
||||
expect(fleet.points.at(-1)).toMatchObject({ reportingHosts: 2, expectedHosts: 2 });
|
||||
const selected = readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch);
|
||||
expect(selected.points.at(-1)).toMatchObject({ reportingHosts: 1, expectedHosts: 1,
|
||||
memoryBytesAvailable: 4 * GiB - 500_000_000 });
|
||||
const missing = readCapacityHistory({ ...query, hostId: 'unknown' }, epoch);
|
||||
expect(missing.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true);
|
||||
});
|
||||
|
||||
it('excludes invalid telemetry and keeps measured exhaustion as zero rather than a gap', async () => {
|
||||
await host('one');
|
||||
await sample('one', epoch - 45_000, 100, 100_000_000);
|
||||
await sample('one', epoch - 30_000, -1);
|
||||
await sample('one', epoch - 15_000, 10, 17 * GiB);
|
||||
await sample('one', epoch - 90_000, 101);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory(query, epoch);
|
||||
expect(history.points.at(-1)).toMatchObject({ cpuCoresAvailable: 0, memoryBytesAvailable: 0, reportingHosts: 1 });
|
||||
expect(history.points.at(-2)).toMatchObject({ cpuCoresAvailable: null, memoryBytesAvailable: null, reportingHosts: 0 });
|
||||
});
|
||||
|
||||
it('bounds long ranges and clips samples outside the interval including future telemetry', async () => {
|
||||
await host('one');
|
||||
await sample('one', epoch - 7 * 86_400_000 - 1);
|
||||
await sample('one', epoch - 7 * 86_400_000, 0);
|
||||
await sample('one', epoch);
|
||||
await sample('one', epoch + 1, 100);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory({ ...query, range: '7d' }, epoch);
|
||||
expect(history.points).toHaveLength(168);
|
||||
expect(history.bucketMs).toBe(3_600_000);
|
||||
expect(history.points[0].cpuCoresAvailable).toBeCloseTo(7.6);
|
||||
expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(5.6);
|
||||
expect(readCapacityHistory({ ...query, range: '24h' }, epoch).points).toHaveLength(144);
|
||||
});
|
||||
|
||||
it('reads history on a separate worker and refreshes its bounded cache after 30 seconds', async () => {
|
||||
await host('one'); await sample('one', epoch - 15_000);
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(epoch);
|
||||
const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js');
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const first = capacityHistoryReport(query);
|
||||
const winner = await Promise.race([first.then(() => 'report'),
|
||||
new Promise<string>(resolve => setTimeout(() => resolve('timer'), 0))]);
|
||||
expect(winner).toBe('timer');
|
||||
expect(await first).toEqual(readCapacityHistory(query, epoch));
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE ec_host_resource_samples SET cpu_usage_percent=100').run();
|
||||
expect(await capacityHistoryReport(query)).toEqual(await first);
|
||||
now.mockReturnValue(epoch + 30_000);
|
||||
expect((await capacityHistoryReport(query)).points.at(-1)?.cpuCoresAvailable).toBe(0);
|
||||
});
|
||||
|
||||
it('opens the history worker on a fresh database before any host has enrolled', async () => {
|
||||
const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js');
|
||||
const history = await capacityHistoryReport(query);
|
||||
expect(history.points).toHaveLength(60);
|
||||
expect(history.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import type { CapacityData, CapacityQuery } from '../../src/lib/capacity.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
let hostId: string;
|
||||
const GiB = 1024 ** 3;
|
||||
const epoch = Date.parse('2026-10-05T12:00:00Z');
|
||||
const image = 'capacity-test-runtime';
|
||||
const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' };
|
||||
const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB,
|
||||
cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
|
||||
runtimeImages: [image], libraries: [{ id: 't5-version', state: 'ready' }, { id: 't6-version', state: 'ready' }] };
|
||||
|
||||
async function settle() { await setImmediate(); await setImmediate(); }
|
||||
async function addHost(name: string, overrides = {}) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name, portMin: 28960, portMax: 28962 });
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
||||
hosts.heartbeat(host.id, { ...inventory, ...overrides });
|
||||
await settle();
|
||||
return host.id;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image);
|
||||
fixture = await newFixture([]);
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js');
|
||||
ensureGameFileSchema(db());
|
||||
for (const game of ['t5', 't6']) {
|
||||
db().prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)')
|
||||
.run(`${game}-library`, game, game, '', new Date(epoch).toISOString());
|
||||
db().prepare(`INSERT INTO game_file_versions
|
||||
(id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,'','[]','ready',?,?,'','','')`)
|
||||
.run(`${game}-version`, `${game}-library`, '1', new Date(epoch).toISOString(), new Date(epoch).toISOString());
|
||||
}
|
||||
hostId = await addHost('Main host');
|
||||
});
|
||||
afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); await fixture.cleanup(); });
|
||||
|
||||
describe('capacity overview', () => {
|
||||
it('shares the cached launch estimate and counts a dual-game host only once without placement or history reads', async () => {
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
|
||||
const placement = await import('../../src/lib/server/hosts/capacity.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const choose = vi.spyOn(placement, 'choosePlacement');
|
||||
const prepare = vi.spyOn(db(), 'prepare');
|
||||
const current = await readCapacityCurrent(query);
|
||||
const launch = launchAvailability(Array.from({ length: 20 }, (_, index) => ({
|
||||
image, libraryId: index % 2 ? 't5-version' : 't6-version', automatic: true
|
||||
}))).availability;
|
||||
expect(current.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1,
|
||||
portsAvailable: 3, cpuCoresAvailable: launch.cpuCoresAvailable, memoryBytesAvailable: launch.memoryBytesAvailable });
|
||||
expect(current.summary.estimatedServers).toBe(launch.estimatedAvailable);
|
||||
expect(current.hosts[0]).toMatchObject({ games: ['t5', 't6'], limitingResource: 'Ports', calibration: {
|
||||
running: { source: 'bootstrap' }, startup: { source: 'bootstrap' }
|
||||
} });
|
||||
expect(choose).not.toHaveBeenCalled();
|
||||
expect(prepare.mock.calls.some(([sql]) => /ec_host_resource_samples|ec_instance_resource_samples|native_instances/.test(sql))).toBe(false);
|
||||
});
|
||||
|
||||
it('filters launch eligibility by game and host while keeping all hosts available in the selector', async () => {
|
||||
const t5Only = await addHost('T5 only', { libraries: [{ id: 't5-version', state: 'ready' }] });
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).summary).toMatchObject({ estimatedServers: 6, eligibleHosts: 2 });
|
||||
const t6 = await readCapacityCurrent({ ...query, game: 't6' });
|
||||
expect(t6.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 2 });
|
||||
expect(t6.hosts.find(host => host.id === t5Only)?.reasons.join(' ')).toContain('Latest game files');
|
||||
const selected = await readCapacityCurrent({ ...query, game: 't5', hostId: t5Only });
|
||||
expect(selected.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1 });
|
||||
expect(selected.hosts.map(host => host.id)).toEqual([t5Only]);
|
||||
expect(selected.hostOptions.map(host => host.id)).toEqual([hostId, t5Only]);
|
||||
});
|
||||
|
||||
it('excludes stale and disabled hosts and does not show stale resource readings as current', async () => {
|
||||
const other = await addHost('Disabled host');
|
||||
const { updateHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
updateHost(other, { mode: 'disabled' }); await settle();
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).summary.eligibleHosts).toBe(1);
|
||||
vi.setSystemTime(epoch + 91_000);
|
||||
heartbeat(other, inventory);
|
||||
const current = await readCapacityCurrent(query);
|
||||
expect(current.summary).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, memoryBytesAvailable: 0,
|
||||
portsAvailable: 0, eligibleHosts: 0, totalHosts: 2 });
|
||||
expect(current.hosts.find(host => host.id === hostId)).toMatchObject({ connected: false, eligible: false,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
expect(current.hosts.find(host => host.id === other)?.reasons.join(' ')).toContain('disabled');
|
||||
});
|
||||
|
||||
it('preserves unknown measurements and distinguishes them from measured exhaustion', async () => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
heartbeat(hostId, { ...inventory, cpuUsagePercent: undefined, memoryAvailable: 65 * GiB });
|
||||
const unknown = await readCapacityCurrent(query);
|
||||
expect(unknown.summary).toMatchObject({ estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null,
|
||||
eligibleHosts: 1, portsAvailable: 3 });
|
||||
expect(unknown.hosts[0].limitingResource).toBeNull();
|
||||
expect(unknown.warnings.join(' ')).toContain('unknown');
|
||||
heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 });
|
||||
expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, limitingResource: 'CPU' });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['CPU', { cpuUsagePercent: 90 }, 1],
|
||||
['Memory', { memoryAvailable: GiB + 500_000_000 }, 1],
|
||||
['Ports', {}, 3]
|
||||
])('identifies %s as the resource limiting additional servers', async (limitingResource, overrides, estimatedServers) => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory, ...overrides });
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ limitingResource, estimatedServers });
|
||||
});
|
||||
|
||||
it('reserves OS headroom and unmeasured startups without double-counting TCP and UDP ports', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js');
|
||||
const deployment = beginDeployment('t5_zombie_theater_1', 'create');
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('starting','t5_zombie_theater_1',?,'t5-version',?,28960,?)`).run(hostId, image, new Date(epoch).toISOString());
|
||||
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deployment, 'starting');
|
||||
reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'udp' });
|
||||
reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'tcp' });
|
||||
heartbeat(hostId, inventory);
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
const current = await readCapacityCurrent(query);
|
||||
expect(current.summary).toMatchObject({ cpuCoresAvailable: 14.7, memoryBytesAvailable: 59 * GiB - 500_000_000,
|
||||
portsAvailable: 2, activeStartups: 1 });
|
||||
heartbeat(hostId, { ...inventory, instanceResources: [{ instanceId: 'starting', generation: 1,
|
||||
uptimeSeconds: 30, cpuCores: 0.1, memoryBytes: GiB / 4 }] });
|
||||
expect((await readCapacityCurrent(query)).summary).toMatchObject({ cpuCoresAvailable: 15.2,
|
||||
memoryBytesAvailable: 60 * GiB - 500_000_000, portsAvailable: 2 });
|
||||
});
|
||||
|
||||
it('serves non-cacheable filtered history and current data and rejects unknown hosts', async () => {
|
||||
const { GET } = await import('../../src/routes/api/capacity/+server.js');
|
||||
const response = await GET({ url: new URL(`https://ec.test/api/capacity?range=1h&game=t6&host=${hostId}`) });
|
||||
expect(response.headers.get('cache-control')).toBe('no-store');
|
||||
const result = await response.json() as CapacityData;
|
||||
expect(result.query).toEqual({ ...query, game: 't6', hostId });
|
||||
expect(result.current.hosts.map(host => host.id)).toEqual([hostId]);
|
||||
expect(result.history).toMatchObject({ range: '1h', bucketMs: 60_000 });
|
||||
expect(result.history.points).toHaveLength(60);
|
||||
const fallback = await GET({ url: new URL('https://ec.test/api/capacity?range=unbounded&game=invalid') });
|
||||
expect((await fallback.json()).query).toEqual({ range: '24h', game: 'all', hostId: null });
|
||||
await expect(GET({ url: new URL('https://ec.test/api/capacity?host=missing') })).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user