Add autoscaling policy model and deterministic planner

src/lib/server/autoscaling/planner.ts

@@ -0,0 +1,42 @@
+import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+
+export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
+/** Only distinct, continuous evidence can advance an empty interval. */
+export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
+    const stamp = member.observedAt;
+    const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
+    const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
+        && previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
+        && now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
+    return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
+}
+
+export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
+    const { members } = snapshot;
+    const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
+    const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
+    const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
+    const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
+    const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
+        ({ action, reason, server, desired, occupied, empty, pending, unknown });
+    if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
+    if (policy.pausedReason) return result('wait', policy.pausedReason);
+    if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
+    const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
+    if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
+    if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
+    if (policy.nextActionAt > now) return result('wait', 'Action cooldown is active.');
+    if (occupied + empty < desired) {
+        if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
+        const available = snapshot.keyedNames.filter(name => !snapshot.reservedNames.includes(name)).sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
+        return available.length ? result('create', 'Replenish empty lobby capacity.', available[0]) : result('wait', 'No unused lobby identity has a saved key.');
+    }
+    if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
+        const candidates = members.filter(m => m.state === 'empty' && m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000)
+            .sort((a, b) => a.emptySince! - b.emptySince! || a.name.localeCompare(b.name, 'en', { numeric: true }));
+        if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
+        return result('wait', 'Waiting for a continuous empty interval.');
+    }
+    return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
+}

src/lib/server/autoscaling/schema.ts

@@ -0,0 +1,35 @@
+import type Database from 'better-sqlite3';
+
+export function ensureAutoscalingSchema(handle: Database.Database): void {
+    handle.exec(`
+        CREATE TABLE IF NOT EXISTS autoscaling_policies (
+            id TEXT PRIMARY KEY, game TEXT NOT NULL, map_slug TEXT NOT NULL,
+            revision INTEGER NOT NULL DEFAULT 1, settings TEXT NOT NULL,
+            mode TEXT NOT NULL DEFAULT 'off', next_action_at INTEGER NOT NULL DEFAULT 0,
+            paused_reason TEXT, owns_map INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL,
+            UNIQUE(game,map_slug)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_members (
+            policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), server TEXT NOT NULL,
+            identity TEXT NOT NULL, observation TEXT NOT NULL, adopted INTEGER NOT NULL DEFAULT 0,
+            PRIMARY KEY(policy_id,server)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_decisions (
+            id TEXT PRIMARY KEY, policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id),
+            revision INTEGER NOT NULL, evaluated_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL,
+            fingerprint TEXT NOT NULL, decision TEXT NOT NULL, evidence TEXT NOT NULL,
+            operation_id TEXT UNIQUE REFERENCES server_deployments(id), resolved INTEGER NOT NULL DEFAULT 1
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS autoscaling_one_intent ON autoscaling_decisions(policy_id) WHERE resolved=0;
+        CREATE INDEX IF NOT EXISTS autoscaling_history ON autoscaling_decisions(policy_id,evaluated_at DESC);
+        CREATE TABLE IF NOT EXISTS autoscaling_samples (
+            policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), sampled_at INTEGER NOT NULL,
+            mode TEXT NOT NULL, occupied INTEGER NOT NULL, empty INTEGER NOT NULL, unknown INTEGER NOT NULL,
+            deficit INTEGER NOT NULL, PRIMARY KEY(policy_id,sampled_at)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_authorizations (
+            operation_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL,
+            expires_at INTEGER NOT NULL, claimed_at INTEGER, revoked_at INTEGER
+        );
+    `);
+}

src/lib/server/autoscaling/snapshot.ts

@@ -0,0 +1,72 @@
+import { db } from '../db/index.js';
+import { listServers, type ServerRow } from '../fleet/servers.js';
+import { observationReader } from '../fleet/status.js';
+import { parseServerName, zombieMaps } from '../../map-aliases.js';
+import { freshStatusTimestamp } from '../../server-freshness.js';
+import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
+import { emptyObservation, type EmptyObservation } from './planner.js';
+
+export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
+    const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
+    if (!game) return null;
+    if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
+    if (server.map_slug) return null;
+    if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
+    const matches = zombieMaps(game).filter(m => m.canonical === server.map);
+    return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
+}
+
+export function protectedWork(server: string, exceptOperation = ''): string | null {
+    const handle = db();
+    if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
+    if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
+    if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
+    if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
+    return null;
+}
+
+export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
+    const handle = db(), observe = observationReader();
+    const rows = listServers();
+    const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
+    const blockers: string[] = [];
+    for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
+    const operations = handle.prepare(`SELECT id,server,action,state FROM server_deployments WHERE server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string }[];
+    const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
+        JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
+    const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
+        const observation = observe(row), operation = operations.find(o => o.server === row.name);
+        const protection = protectedWork(row.name, exceptOperation);
+        const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
+        const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
+        let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
+        let players = observation.health === 'online' ? observation.last_players : null;
+        let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
+        let source = observation.status_source;
+        // Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
+        if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
+            && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
+            players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
+        }
+        if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
+        else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id} is ${operation.state}`; }
+        else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
+        else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
+        return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null, blocker, operationId: operation?.id ?? null };
+    });
+    for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: 'Creation is pending.', operationId: operations.find(o => o.server === row.name)?.id ?? null });
+    const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
+    for (const member of members) {
+        const saved = retained.find(r => r.server === member.name);
+        const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
+        member.emptySince = emptyObservation(member, previous, now).emptySince;
+    }
+    const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
+    for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
+        reservedNames.add(row.name);
+        if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
+    }
+    const keyedNames = (handle.prepare("SELECT server_id AS name FROM server_keys WHERE length(trim(pluto_key))>0").all() as { name: string }[]).filter(matches).map(r => r.name);
+    return { members, reservedNames: [...reservedNames], keyedNames, blockers };
+}