Automatically place servers using learned resource capacity

src/lib/server/hosts/calibration.ts

@@ -0,0 +1,106 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from './schema.js';
+import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js';
+
+const BOOTSTRAP_CPU = 0.5;
+const BOOTSTRAP_MEMORY = 1024 ** 3;
+const MARGIN = 1.25;
+const WINDOW_MS = 24 * 60 * 60_000;
+const MAX_SAMPLES = 10_000;
+type Phase = 'startup' | 'running';
+interface AllowanceRow {
+    scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
+    window_start: string; observed_at: string; computed_at: string;
+}
+function database() { const handle = db(); ensureHostSchema(handle); return handle; }
+
+export function validateInstanceResources(inventory: HostInventory): void {
+    const samples = inventory.instanceResources;
+    if (samples === undefined) return;
+    if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
+    const seen = new Set<string>();
+    for (const sample of samples) {
+        if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
+            || !Number.isSafeInteger(sample.generation) || sample.generation < 1
+            || !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
+            || typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
+            || !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
+            || (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
+            || (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
+            || seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
+        seen.add(sample.instanceId);
+    }
+}
+
+/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
+export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
+    if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
+    const handle = database();
+    const bucket = timestamp.slice(0, 16) + ':00.000Z';
+    const observed = new Map<Phase, { cpu: number; memory: number }>();
+    const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
+    const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
+        (host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
+        ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
+            cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`);
+    for (const sample of inventory.instanceResources) {
+        // An agent can only train from its currently owned generation; foreign,
+        // retired and delayed pre-rebuild samples cannot contaminate the model.
+        if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
+        const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
+        insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
+        const prior = observed.get(phase);
+        observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
+    }
+    for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
+        const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
+        if (!previous || previous.computed_at < bucket) {
+            const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
+                WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
+                .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
+                { cpu_cores: number; memory_bytes: number; bucket: string }[];
+            const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
+            const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
+            const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
+            handle.prepare(`INSERT INTO ec_resource_allowances
+                (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
+                cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
+                window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
+                .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
+            if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
+                .run(new Date(Date.parse(timestamp) - WINDOW_MS).toISOString());
+        } else {
+            // Raise the allowance immediately on a new peak; a lower sample
+            // cannot reduce it until the next scheduled percentile calculation.
+            handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
+                .run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
+        }
+    }
+}
+
+/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
+export function capacityCalibration(hostId: string): CapacityCalibration {
+    const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
+        .all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
+    const allowance = (phase: Phase): ResourceAllowance => {
+        const local = rows.find(row => row.scope === hostId && row.phase === phase);
+        const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
+        const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
+            && Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
+        const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
+        return {
+            // A few higher observations can increase a fleet/bootstrap allowance
+            // immediately. Decreasing it requires the evidence thresholds above.
+            cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
+            memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
+            source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
+            samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
+            observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
+        };
+    };
+    const running = allowance('running'); const startup = allowance('startup');
+    if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
+    startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
+    startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
+    return { running, startup };
+}