diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 676906b..2b6fdc1 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -162,6 +162,13 @@ headroom**. Each calculation considers up to the latest 10,000 minute peaks. Host-local evidence is preferred to account for different hardware; new hosts can borrow the fleet's learned allowance. Startup allowances are never lower than running allowances. The host page shows both allowances and their source. +Expand **View samples** in Learned server capacity to inspect retained minute +peaks for the last 24 hours. Filter by startup/running phase or switch from this +host to the entire fleet. Rows show time, server and generation, CPU cores, and +memory in MiB before headroom; CPU and memory peaks may come from different +heartbeats. Results are paginated, and **Refresh samples** loads the latest stored +measurements without contacting agents. The allowance count may lag the table +until the next calculation or reflect a different source. Lower allowances require at least three startup minute samples spanning two minutes, or twenty running minute samples spanning five minutes. Until then, diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index 303fbcf..cb2f8cb 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -79,6 +79,13 @@ export interface ResourceAllowance { samples: number; observedAt: string | null; } export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance } +export interface CapacitySample { + hostId: string; hostName: string | null; instanceId: string; server: string | null; + generation: number; phase: 'startup' | 'running'; timestamp: string; cpuCores: number; memoryBytes: number; +} +export interface CapacitySamplePage { + samples: CapacitySample[]; total: number; page: number; pageSize: number; +} export interface Host { id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string; diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index ef948c8..89db95c 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -1,6 +1,6 @@ import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; -import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js'; +import type { CapacityCalibration, CapacitySample, CapacitySamplePage, HostInventory, ResourceAllowance } from '../../hosts.js'; const BOOTSTRAP_CPU = 0.5; const BOOTSTRAP_MEMORY = 1024 ** 3; @@ -14,6 +14,27 @@ interface AllowanceRow { } function database() { const handle = db(); ensureHostSchema(handle); return handle; } +/** Read retained minute peaks without contacting agents or recalculating allowances. */ +export function capacitySamples(hostId: string, options: { scope: 'host' | 'fleet'; phase: 'all' | Phase; page: number }): CapacitySamplePage { + const handle = database(); + const clauses = ['s.bucket >= ?']; + const values: (string | number)[] = [new Date(Date.now() - WINDOW_MS).toISOString()]; + if (options.scope === 'host') { clauses.push('s.host_id = ?'); values.push(hostId); } + if (options.phase !== 'all') { clauses.push('s.phase = ?'); values.push(options.phase); } + const where = clauses.join(' AND '); + const total = (handle.prepare(`SELECT COUNT(*) AS count FROM ec_instance_resource_samples s WHERE ${where}`).get(...values) as { count: number }).count; + const pageSize = 50; + const page = Math.min(options.page, Math.max(1, Math.ceil(total / pageSize))); + const samples = handle.prepare(`SELECT s.host_id AS hostId, h.name AS hostName, s.instance_id AS instanceId, + i.server, s.generation, s.phase, s.bucket AS timestamp, s.cpu_cores AS cpuCores, s.memory_bytes AS memoryBytes + FROM ec_instance_resource_samples s + LEFT JOIN ec_hosts h ON h.id=s.host_id + LEFT JOIN native_instances i ON i.id=s.instance_id AND i.host_id=s.host_id + WHERE ${where} ORDER BY s.bucket DESC, s.host_id, s.instance_id, s.generation, s.phase LIMIT ? OFFSET ?`) + .all(...values, pageSize, (page - 1) * pageSize) as CapacitySample[]; + return { samples, total, page, pageSize }; +} + export function validateInstanceResources(inventory: HostInventory): void { if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string' || inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.'); diff --git a/src/lib/ui/components/CapacitySamples.svelte b/src/lib/ui/components/CapacitySamples.svelte new file mode 100644 index 0000000..dc5acd2 --- /dev/null +++ b/src/lib/ui/components/CapacitySamples.svelte @@ -0,0 +1,84 @@ + + +
+ View samples + {#if open} +

Retained minute peaks from the last 24 hours, before the 25% allowance headroom. CPU and memory peaks may come from different heartbeats. Choose the fleet to inspect measurements from other hosts used by fleet allowances.

+
+ + + +
+ {#if error}{/if} + {#if loading}

Loading samples…

+ {:else if result} + {#if result.total === 0}

No {phase === 'all' ? '' : phase === 'startup' ? 'startup ' : 'running '}samples recorded for this source in the last 24 hours.

+ {:else} +

{result.total.toLocaleString()} samples · newest first. Counts can differ from the allowance above, which is recalculated periodically.

+ +
+ + {#if scope === 'fleet'}{/if} + {#each result.samples as sample} + + {#if scope === 'fleet'}{/if} + + + {/each} +
CPU and memory peaks per container and minute; times are local.
TimeHostServerPhaseCPU coresMemory
{sample.hostName ?? sample.hostId}{sample.server ?? sample.instanceId}Generation {sample.generation}{sample.phase === 'startup' ? 'Starting' : 'Running'}{sample.cpuCores.toFixed(3)}{(sample.memoryBytes / 1048576).toFixed(1)} MiB
+
+ + {/if} + {/if} + {/if} +
+ + diff --git a/src/routes/api/hosts/[id]/samples/+server.ts b/src/routes/api/hosts/[id]/samples/+server.ts new file mode 100644 index 0000000..8619e6d --- /dev/null +++ b/src/routes/api/hosts/[id]/samples/+server.ts @@ -0,0 +1,17 @@ +import { error, json } from '@sveltejs/kit'; +import { getHost } from '$lib/server/hosts/index.js'; +import { requireManagementOrigin } from '$lib/server/hosts/http.js'; +import { capacitySamples } from '$lib/server/hosts/calibration.js'; + +export function GET({ request, params }: { request: Request; params: { id: string } }) { + requireManagementOrigin(request); + if (!getHost(params.id)) error(404, 'Host not found.'); + const search = new URL(request.url).searchParams; + const scope = search.get('scope') ?? 'host'; + const phase = search.get('phase') ?? 'all'; + const page = Number(search.get('page') ?? '1'); + if (scope !== 'host' && scope !== 'fleet') error(400, 'Invalid sample scope.'); + if (phase !== 'all' && phase !== 'startup' && phase !== 'running') error(400, 'Invalid sample phase.'); + if (!Number.isSafeInteger(page) || page < 1) error(400, 'Invalid sample page.'); + return json(capacitySamples(params.id, { scope, phase, page }), { headers: { 'Cache-Control': 'no-store' } }); +} diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index c3e0a71..3da231e 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -10,6 +10,7 @@ import Button from '$ui/components/primitives/Button.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; import HostPreparation from '$ui/components/HostPreparation.svelte'; + import CapacitySamples from '$ui/components/CapacitySamples.svelte'; let host = $state(null); let operations = $state([]); let calibration = $state(null); let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state(''); @@ -64,6 +65,7 @@ {/each} {#if host.inventory.instanceResourcesError}

{host.inventory.instanceResourcesError}

{/if} {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}

Use agent 1.4.6 or later for resource sampling. Measurements arrive with agent heartbeats and are combined into one peak sample per container per minute. The first five minutes after a container starts count toward startup; later measurements count toward running. Temporary allowances stay in use until enough measurements are available.

{/if} + {#key host.id}{/key} {/if}

Inventory

Hostname
{host.hostname || 'Pending'}
Agent
{host.inventory.agentVersion ?? 'Unknown'}
Platform
{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}
CPU
{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)
CPU usage
{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}
Memory available / total
{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}
Game storage available / total
{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}
Docker
{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}
Current operation
{host.inventory.currentOperation ?? 'None reported'}
diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index b629ddc..74ed571 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -38,6 +38,57 @@ async function learned() { } describe('measured instance capacity', () => { + it('lists retained samples with phase filters, host scope, stable ordering, and bounded pages', async () => { + for (let minute = 0; minute < 101; minute++) await report(minute, { uptimeSeconds: minute < 5 ? minute * 60 : 600 }); + const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); + const first = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }); + const second = capacitySamples(hostId, { scope: 'host', phase: 'all', page: 2 }); + expect(first).toMatchObject({ total: 101, page: 1, pageSize: 50 }); + expect(first.samples).toHaveLength(50); + expect(first.samples[0]).toMatchObject({ hostId, hostName: 'First', instanceId: sample.instanceId, + server: 't5_zombie_theater_instance-1', generation: 1, phase: 'running', + cpuCores: 0.1, memoryBytes: GiB / 4, timestamp: new Date(epoch + 100 * 60_000).toISOString() }); + expect(new Set([...first.samples, ...second.samples].map(row => row.timestamp)).size).toBe(100); + expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 999 })).toMatchObject({ page: 3, samples: [expect.any(Object)] }); + const startup = capacitySamples(hostId, { scope: 'host', phase: 'startup', page: 1 }); + expect(startup.total).toBe(5); + expect(startup.samples.every(row => row.phase === 'startup')).toBe(true); + expect(capacitySamples(hostId, { scope: 'host', phase: 'running', page: 1 }).total).toBe(96); + vi.setSystemTime(epoch + 26 * 60 * 60_000); + expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 })).toMatchObject({ total: 0, samples: [] }); + }); + + it('supports fleet samples and retains instance identity after retirement', async () => { + await report(0); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const other = hosts.createHost({ name: 'Second' }); + hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); + await instance(other.id, 'instance-2'); + await report(1, { instanceId: 'instance-2' }, other.id); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId); + const { capacitySamples } = await import('../../src/lib/server/hosts/calibration.js'); + expect(capacitySamples(hostId, { scope: 'host', phase: 'all', page: 1 }).samples.map(row => row.instanceId)).toEqual([sample.instanceId]); + const fleet = capacitySamples(hostId, { scope: 'fleet', phase: 'all', page: 1 }); + expect(fleet.total).toBe(2); + expect(fleet.samples.map(row => row.hostName)).toEqual(['Second', 'First']); + }); + + it('validates the samples API and rejects agent credentials and cross-origin reads', async () => { + await report(0); + const { GET } = await import('../../src/routes/api/hosts/[id]/samples/+server.js'); + const request = (query = '', headers = {}, id = hostId) => GET({ request: new Request(`https://ec.test/api/hosts/${id}/samples${query}`, { headers }), params: { id } }); + const response = request(); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect(await response.json()).toMatchObject({ total: 1, samples: [expect.objectContaining({ cpuCores: 0.1 })] }); + for (const query of ['?scope=wrong', '?phase=wrong', '?page=-1', '?page=1.5', '?page=NaN', '?page=Infinity']) { + expect(() => request(query)).toThrow(expect.objectContaining({ status: 400 })); + } + expect(() => request('', {}, 'missing')).toThrow(expect.objectContaining({ status: 404 })); + expect(() => request('', { origin: 'https://other.test' })).toThrow(expect.objectContaining({ status: 403 })); + expect(() => request('', { authorization: 'Bearer ec_host_test' })).toThrow(expect.objectContaining({ status: 403 })); + }); + it('reports sampling failure without inventing measurements and clears it after recovery', async () => { const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');