Add capacity overview with host resource history
This commit is contained in:
@@ -7,7 +7,7 @@ const allows = (role: UserRole, route: string | null, method = 'GET') => roleAll
|
||||
|
||||
describe('management route permissions', () => {
|
||||
it.each([
|
||||
'/', '/api/fleet', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/', '/api/fleet', '/capacity', '/api/capacity', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/api/metrics', '/metrics', '/api/matches', '/matches/[id]', '/api/leaderboard',
|
||||
'/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]',
|
||||
'/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events',
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import type { CapacityQuery } from '../../src/lib/capacity.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const epoch = Date.UTC(2026, 9, 5, 12);
|
||||
const GiB = 1024 ** 3;
|
||||
const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' };
|
||||
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
|
||||
|
||||
async function host(id: string, createdAt = epoch - 7 * 86_400_000) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
||||
ensureHostSchema(db());
|
||||
db().prepare('INSERT INTO ec_hosts (id,name,created_at) VALUES (?,?,?)').run(id, id, new Date(createdAt).toISOString());
|
||||
}
|
||||
|
||||
async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores = 8) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO ec_host_resource_samples
|
||||
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES (?,?,?,?,?,?)`)
|
||||
.run(id, new Date(at).toISOString(), cores, cpu, 16 * GiB, memory);
|
||||
}
|
||||
|
||||
describe('capacity history', () => {
|
||||
it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => {
|
||||
await host('one'); await host('two');
|
||||
await sample('one', epoch - 55_000, 0, 8 * GiB);
|
||||
await sample('one', epoch - 30_000, 50, 4 * GiB);
|
||||
await sample('two', epoch - 45_000, 50, 2 * GiB, 4);
|
||||
await sample('one', epoch - 90_000);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory(query, epoch);
|
||||
expect(history.points).toHaveLength(60);
|
||||
expect(history.points.at(-1)).toMatchObject({ at: epoch - 60_000,
|
||||
memoryBytesAvailable: 8 * GiB - 1_000_000_000,
|
||||
reportingHosts: 2, expectedHosts: 2 });
|
||||
expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(7.4);
|
||||
expect(history.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 2,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
expect(history.points[0]).toMatchObject({ reportingHosts: 0, expectedHosts: 2,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
});
|
||||
|
||||
it('scopes history to the selected registered host and does not add hosts before they existed', async () => {
|
||||
await host('one'); await host('two', epoch - 30_000); await host('removed');
|
||||
await sample('one', epoch - 90_000);
|
||||
await sample('one', epoch - 15_000);
|
||||
await sample('two', epoch - 15_000);
|
||||
await sample('removed', epoch - 15_000);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?').run(new Date(epoch).toISOString(), 'removed');
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const fleet = readCapacityHistory(query, epoch);
|
||||
expect(fleet.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 1 });
|
||||
expect(fleet.points.at(-1)).toMatchObject({ reportingHosts: 2, expectedHosts: 2 });
|
||||
const selected = readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch);
|
||||
expect(selected.points.at(-1)).toMatchObject({ reportingHosts: 1, expectedHosts: 1,
|
||||
memoryBytesAvailable: 4 * GiB - 500_000_000 });
|
||||
const missing = readCapacityHistory({ ...query, hostId: 'unknown' }, epoch);
|
||||
expect(missing.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true);
|
||||
});
|
||||
|
||||
it('excludes invalid telemetry and keeps measured exhaustion as zero rather than a gap', async () => {
|
||||
await host('one');
|
||||
await sample('one', epoch - 45_000, 100, 100_000_000);
|
||||
await sample('one', epoch - 30_000, -1);
|
||||
await sample('one', epoch - 15_000, 10, 17 * GiB);
|
||||
await sample('one', epoch - 90_000, 101);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory(query, epoch);
|
||||
expect(history.points.at(-1)).toMatchObject({ cpuCoresAvailable: 0, memoryBytesAvailable: 0, reportingHosts: 1 });
|
||||
expect(history.points.at(-2)).toMatchObject({ cpuCoresAvailable: null, memoryBytesAvailable: null, reportingHosts: 0 });
|
||||
});
|
||||
|
||||
it('bounds long ranges and clips samples outside the interval including future telemetry', async () => {
|
||||
await host('one');
|
||||
await sample('one', epoch - 7 * 86_400_000 - 1);
|
||||
await sample('one', epoch - 7 * 86_400_000, 0);
|
||||
await sample('one', epoch);
|
||||
await sample('one', epoch + 1, 100);
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const history = readCapacityHistory({ ...query, range: '7d' }, epoch);
|
||||
expect(history.points).toHaveLength(168);
|
||||
expect(history.bucketMs).toBe(3_600_000);
|
||||
expect(history.points[0].cpuCoresAvailable).toBeCloseTo(7.6);
|
||||
expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(5.6);
|
||||
expect(readCapacityHistory({ ...query, range: '24h' }, epoch).points).toHaveLength(144);
|
||||
});
|
||||
|
||||
it('reads history on a separate worker and refreshes its bounded cache after 30 seconds', async () => {
|
||||
await host('one'); await sample('one', epoch - 15_000);
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(epoch);
|
||||
const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js');
|
||||
const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js');
|
||||
const first = capacityHistoryReport(query);
|
||||
const winner = await Promise.race([first.then(() => 'report'),
|
||||
new Promise<string>(resolve => setTimeout(() => resolve('timer'), 0))]);
|
||||
expect(winner).toBe('timer');
|
||||
expect(await first).toEqual(readCapacityHistory(query, epoch));
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE ec_host_resource_samples SET cpu_usage_percent=100').run();
|
||||
expect(await capacityHistoryReport(query)).toEqual(await first);
|
||||
now.mockReturnValue(epoch + 30_000);
|
||||
expect((await capacityHistoryReport(query)).points.at(-1)?.cpuCoresAvailable).toBe(0);
|
||||
});
|
||||
|
||||
it('opens the history worker on a fresh database before any host has enrolled', async () => {
|
||||
const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js');
|
||||
const history = await capacityHistoryReport(query);
|
||||
expect(history.points).toHaveLength(60);
|
||||
expect(history.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import type { CapacityData, CapacityQuery } from '../../src/lib/capacity.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
let hostId: string;
|
||||
const GiB = 1024 ** 3;
|
||||
const epoch = Date.parse('2026-10-05T12:00:00Z');
|
||||
const image = 'capacity-test-runtime';
|
||||
const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' };
|
||||
const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB,
|
||||
cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
|
||||
runtimeImages: [image], libraries: [{ id: 't5-version', state: 'ready' }, { id: 't6-version', state: 'ready' }] };
|
||||
|
||||
async function settle() { await setImmediate(); await setImmediate(); }
|
||||
async function addHost(name: string, overrides = {}) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name, portMin: 28960, portMax: 28962 });
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
||||
hosts.heartbeat(host.id, { ...inventory, ...overrides });
|
||||
await settle();
|
||||
return host.id;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image);
|
||||
fixture = await newFixture([]);
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js');
|
||||
ensureGameFileSchema(db());
|
||||
for (const game of ['t5', 't6']) {
|
||||
db().prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)')
|
||||
.run(`${game}-library`, game, game, '', new Date(epoch).toISOString());
|
||||
db().prepare(`INSERT INTO game_file_versions
|
||||
(id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,'','[]','ready',?,?,'','','')`)
|
||||
.run(`${game}-version`, `${game}-library`, '1', new Date(epoch).toISOString(), new Date(epoch).toISOString());
|
||||
}
|
||||
hostId = await addHost('Main host');
|
||||
});
|
||||
afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); await fixture.cleanup(); });
|
||||
|
||||
describe('capacity overview', () => {
|
||||
it('shares the cached launch estimate and counts a dual-game host only once without placement or history reads', async () => {
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
|
||||
const placement = await import('../../src/lib/server/hosts/capacity.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const choose = vi.spyOn(placement, 'choosePlacement');
|
||||
const prepare = vi.spyOn(db(), 'prepare');
|
||||
const current = await readCapacityCurrent(query);
|
||||
const launch = launchAvailability(Array.from({ length: 20 }, (_, index) => ({
|
||||
image, libraryId: index % 2 ? 't5-version' : 't6-version', automatic: true
|
||||
}))).availability;
|
||||
expect(current.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1,
|
||||
portsAvailable: 3, cpuCoresAvailable: launch.cpuCoresAvailable, memoryBytesAvailable: launch.memoryBytesAvailable });
|
||||
expect(current.summary.estimatedServers).toBe(launch.estimatedAvailable);
|
||||
expect(current.hosts[0]).toMatchObject({ games: ['t5', 't6'], limitingResource: 'Ports', calibration: {
|
||||
running: { source: 'bootstrap' }, startup: { source: 'bootstrap' }
|
||||
} });
|
||||
expect(choose).not.toHaveBeenCalled();
|
||||
expect(prepare.mock.calls.some(([sql]) => /ec_host_resource_samples|ec_instance_resource_samples|native_instances/.test(sql))).toBe(false);
|
||||
});
|
||||
|
||||
it('filters launch eligibility by game and host while keeping all hosts available in the selector', async () => {
|
||||
const t5Only = await addHost('T5 only', { libraries: [{ id: 't5-version', state: 'ready' }] });
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).summary).toMatchObject({ estimatedServers: 6, eligibleHosts: 2 });
|
||||
const t6 = await readCapacityCurrent({ ...query, game: 't6' });
|
||||
expect(t6.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 2 });
|
||||
expect(t6.hosts.find(host => host.id === t5Only)?.reasons.join(' ')).toContain('Latest game files');
|
||||
const selected = await readCapacityCurrent({ ...query, game: 't5', hostId: t5Only });
|
||||
expect(selected.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1 });
|
||||
expect(selected.hosts.map(host => host.id)).toEqual([t5Only]);
|
||||
expect(selected.hostOptions.map(host => host.id)).toEqual([hostId, t5Only]);
|
||||
});
|
||||
|
||||
it('excludes stale and disabled hosts and does not show stale resource readings as current', async () => {
|
||||
const other = await addHost('Disabled host');
|
||||
const { updateHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
updateHost(other, { mode: 'disabled' }); await settle();
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).summary.eligibleHosts).toBe(1);
|
||||
vi.setSystemTime(epoch + 91_000);
|
||||
heartbeat(other, inventory);
|
||||
const current = await readCapacityCurrent(query);
|
||||
expect(current.summary).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, memoryBytesAvailable: 0,
|
||||
portsAvailable: 0, eligibleHosts: 0, totalHosts: 2 });
|
||||
expect(current.hosts.find(host => host.id === hostId)).toMatchObject({ connected: false, eligible: false,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null });
|
||||
expect(current.hosts.find(host => host.id === other)?.reasons.join(' ')).toContain('disabled');
|
||||
});
|
||||
|
||||
it('preserves unknown measurements and distinguishes them from measured exhaustion', async () => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
heartbeat(hostId, { ...inventory, cpuUsagePercent: undefined, memoryAvailable: 65 * GiB });
|
||||
const unknown = await readCapacityCurrent(query);
|
||||
expect(unknown.summary).toMatchObject({ estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null,
|
||||
eligibleHosts: 1, portsAvailable: 3 });
|
||||
expect(unknown.hosts[0].limitingResource).toBeNull();
|
||||
expect(unknown.warnings.join(' ')).toContain('unknown');
|
||||
heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 });
|
||||
expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, limitingResource: 'CPU' });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['CPU', { cpuUsagePercent: 90 }, 1],
|
||||
['Memory', { memoryAvailable: GiB + 500_000_000 }, 1],
|
||||
['Ports', {}, 3]
|
||||
])('identifies %s as the resource limiting additional servers', async (limitingResource, overrides, estimatedServers) => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory, ...overrides });
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ limitingResource, estimatedServers });
|
||||
});
|
||||
|
||||
it('reserves OS headroom and unmeasured startups without double-counting TCP and UDP ports', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js');
|
||||
const deployment = beginDeployment('t5_zombie_theater_1', 'create');
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('starting','t5_zombie_theater_1',?,'t5-version',?,28960,?)`).run(hostId, image, new Date(epoch).toISOString());
|
||||
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deployment, 'starting');
|
||||
reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'udp' });
|
||||
reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'tcp' });
|
||||
heartbeat(hostId, inventory);
|
||||
const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js');
|
||||
const current = await readCapacityCurrent(query);
|
||||
expect(current.summary).toMatchObject({ cpuCoresAvailable: 14.7, memoryBytesAvailable: 59 * GiB - 500_000_000,
|
||||
portsAvailable: 2, activeStartups: 1 });
|
||||
heartbeat(hostId, { ...inventory, instanceResources: [{ instanceId: 'starting', generation: 1,
|
||||
uptimeSeconds: 30, cpuCores: 0.1, memoryBytes: GiB / 4 }] });
|
||||
expect((await readCapacityCurrent(query)).summary).toMatchObject({ cpuCoresAvailable: 15.2,
|
||||
memoryBytesAvailable: 60 * GiB - 500_000_000, portsAvailable: 2 });
|
||||
});
|
||||
|
||||
it('serves non-cacheable filtered history and current data and rejects unknown hosts', async () => {
|
||||
const { GET } = await import('../../src/routes/api/capacity/+server.js');
|
||||
const response = await GET({ url: new URL(`https://ec.test/api/capacity?range=1h&game=t6&host=${hostId}`) });
|
||||
expect(response.headers.get('cache-control')).toBe('no-store');
|
||||
const result = await response.json() as CapacityData;
|
||||
expect(result.query).toEqual({ ...query, game: 't6', hostId });
|
||||
expect(result.current.hosts.map(host => host.id)).toEqual([hostId]);
|
||||
expect(result.history).toMatchObject({ range: '1h', bucketMs: 60_000 });
|
||||
expect(result.history.points).toHaveLength(60);
|
||||
const fallback = await GET({ url: new URL('https://ec.test/api/capacity?range=unbounded&game=invalid') });
|
||||
expect((await fallback.json()).query).toEqual({ range: '24h', game: 'all', hostId: null });
|
||||
await expect(GET({ url: new URL('https://ec.test/api/capacity?host=missing') })).rejects.toMatchObject({ status: 404 });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user