From 36cf2b9c055e07852f6ae57b18724ac289f41af3 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 22:52:50 -0400 Subject: [PATCH] Add autoscaling dry run and policy controls --- src/lib/autoscaling.ts | 7 ++ src/lib/navigation.ts | 1 + src/lib/server/autoscaling/index.ts | 97 ++++++++++++++++++ src/lib/server/pipeline/worker.ts | 6 +- src/routes/+page.svelte | 1 + src/routes/api/autoscaling/+server.ts | 15 +++ src/routes/autoscaling/+page.server.ts | 2 + src/routes/autoscaling/+page.svelte | 133 +++++++++++++++++++++++++ src/routes/collections/+page.svelte | 2 +- src/routes/metrics/+page.svelte | 2 +- tests/unit/autoscaling-dry-run.test.ts | 55 ++++++++++ 11 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 src/lib/server/autoscaling/index.ts create mode 100644 src/routes/api/autoscaling/+server.ts create mode 100644 src/routes/autoscaling/+page.server.ts create mode 100644 src/routes/autoscaling/+page.svelte create mode 100644 tests/unit/autoscaling-dry-run.test.ts diff --git a/src/lib/autoscaling.ts b/src/lib/autoscaling.ts index 6fdec5e..507f690 100644 --- a/src/lib/autoscaling.ts +++ b/src/lib/autoscaling.ts @@ -21,6 +21,13 @@ export interface ScalingDecision { action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null; desired: number; occupied: number; empty: number; pending: number; unknown: number; } +export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] } +export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null } +export interface ScalingView extends ScalingPreview { + history: ScalingHistory[]; lastEvaluatedAt: number | null; + totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number }; +} +export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean } export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy'; export function scalingSettings(input: unknown): ScalingSettings { diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 7413c93..38d2821 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -7,6 +7,7 @@ export const navigationGroups: NavigationGroup[] = [ { id: 'fleet', label: 'Fleet', items: [ { label: 'Overview', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') }, { label: 'Collections', href: '/collections' }, + { label: 'Autoscaling', href: '/autoscaling' }, { label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) }, { label: 'Capacity', href: '/capacity' } ] }, diff --git a/src/lib/server/autoscaling/index.ts b/src/lib/server/autoscaling/index.ts new file mode 100644 index 0000000..0f6dfda --- /dev/null +++ b/src/lib/server/autoscaling/index.ts @@ -0,0 +1,97 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { db } from '../db/index.js'; +import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js'; +import { emptyObservation, planScaling } from './planner.js'; +import { scalingSnapshot } from './snapshot.js'; + +export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } } +interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string } +const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at }); +export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present); +export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); } +export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1'; + +export function previewScaling(input: unknown, id?: string, revision?: number): ScalingPreview { + const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined; + if (id && !existing) throw new ScalingError('Policy not found.', 404); + if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409); + if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.'); + const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0, + nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' }; + const snapshot = scalingSnapshot(policy); + const blockers = liveScaling(policy.mode) ? ['Live autoscaling is not available yet.'] : []; + const decision = planScaling(policy, snapshot, Date.now()); + const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(), + members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex'); + return { policy, snapshot, decision, token, blockers }; +} + +export function saveScaling(input: unknown, id?: string, revision?: number, token?: string): ScalingPolicy { + return db().transaction(() => { + const preview = previewScaling(input, id, revision); + if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409); + if (preview.blockers.length) throw new ScalingError(preview.blockers[0]); + const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString(); + if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409); + db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at); + // An edit starts a new observation interval; earlier inactivity is not authorization. + db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId); + return getScalingPolicy(nextId)!; + })(); +} + +export function setScalingGlobalPause(paused: boolean): void { + db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0'); +} + +export async function runAutoscaling(): Promise { + const now = Date.now(); + for (const policy of listScalingPolicies()) { + if (policy.mode === 'off') continue; + db().transaction(() => { + const snapshot = scalingSnapshot(policy, now); + if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); + const decision = planScaling(policy, snapshot, now); + for (const member of snapshot.members) { + const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now); + db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?) + ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation)); + } + // Evidence includes provenance and timestamps but never keys or player rosters. + const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers }); + const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex'); + const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined; + if (last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); + else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)') + .run(randomUUID(), policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence); + db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`) + .run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty)); + })(); + } +} + +export function pruneAutoscaling(now = Date.now()): void { + const cutoff = now - 31 * 86400_000; + db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at { + const snapshot = scalingSnapshot(policy); + if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); + const decision = planScaling(policy, snapshot, Date.now()); + const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as + { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[]) + .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id })); + // Integrate only bounded intervals between adjacent samples; outages contribute no invented time. + const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration + FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes, + COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours, + COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes, + COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals']; + return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals }; + }); + return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: false, removalAvailable: false }; +} diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index dcc127a..40215eb 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -1,4 +1,5 @@ import { pruneHistory, pruneSampleHistory } from '../retention.js'; +import { runAutoscaling, pruneAutoscaling } from '../autoscaling/index.js'; import { config } from '../config.js'; import { processProvisioning } from '../provisioning/worker.js'; import { sweep } from './sweep.js'; @@ -54,8 +55,9 @@ function reconcileRequesters(): void { ['campaign', 'Reconciling campaigns', sweep], ['beta', 'Reconciling beta tests', betaTick], ['maintenance', 'Checking maintenance and recovery', runMaintenance], + ['autoscaling', 'Evaluating lobby autoscaling', runAutoscaling], ['game-files', 'Removing unused game files', cleanupControllerLibraries], - ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); }] + ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); }] ]; for (const [key, label, work] of requesters) { if (requesterFlights.has(key)) continue; @@ -65,7 +67,7 @@ function reconcileRequesters(): void { } const flight = trackActivity(key, label, work) .catch(err => { console.error(`[worker] ${key} error:`, err); }) - .then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); }) + .then(async result => { if (key !== 'game-files' && key !== 'retention' && (key !== 'autoscaling' || result === true)) await tick(true); }) .finally(() => { requesterFlights.delete(key); }); requesterFlights.set(key, flight); } diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index f537d31..97a46ad 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -143,6 +143,7 @@
{#snippet actions()} + {#if canOperate}{/if} {/snippet} diff --git a/src/routes/api/autoscaling/+server.ts b/src/routes/api/autoscaling/+server.ts new file mode 100644 index 0000000..fbce1ca --- /dev/null +++ b/src/routes/api/autoscaling/+server.ts @@ -0,0 +1,15 @@ +import { json } from '@sveltejs/kit'; +import { previewScaling, saveScaling, scalingReport, setScalingGlobalPause, ScalingError } from '$lib/server/autoscaling/index.js'; + +export function GET() { return json(scalingReport()); } +export async function POST({ request }: { request: Request }) { + try { + const body = await request.json(); + if (!body || typeof body !== 'object' || Array.isArray(body)) throw new ScalingError('Provide a policy action.'); + if (body.id !== undefined && typeof body.id !== 'string') throw new ScalingError('Invalid policy identity.'); + if (body.action === 'preview') return json(previewScaling(body.settings, body.id, body.revision)); + if (body.action === 'save') return json({ policy: saveScaling(body.settings, body.id, body.revision, body.token) }); + if (body.action === 'global-pause' && typeof body.paused === 'boolean') { setScalingGlobalPause(body.paused); return json({ ok: true }); } + throw new ScalingError('Choose a supported policy action.'); + } catch (error) { return json({ error: (error as Error).message }, { status: error instanceof ScalingError ? error.status : 400 }); } +} diff --git a/src/routes/autoscaling/+page.server.ts b/src/routes/autoscaling/+page.server.ts new file mode 100644 index 0000000..e7a1f0c --- /dev/null +++ b/src/routes/autoscaling/+page.server.ts @@ -0,0 +1,2 @@ +import { scalingReport } from '$lib/server/autoscaling/index.js'; +export function load() { return { autoscaling: scalingReport() }; } diff --git a/src/routes/autoscaling/+page.svelte b/src/routes/autoscaling/+page.svelte new file mode 100644 index 0000000..46a0fe6 --- /dev/null +++ b/src/routes/autoscaling/+page.svelte @@ -0,0 +1,133 @@ + + +Autoscaling | Electric Cherry +
+ + {#snippet actions()}{#if canOperate}{/if}{/snippet} + +

Fleet · Collections · Metrics

+ {#if report.globalPaused}{/if} +

Start with a two-week dry run. Dry run records hypothetical actions and reserves no lobby or host capacity.

+ {#if refreshError}

{refreshError}

{/if} + {#if actionError}

{actionError}

{/if} + {#if notice}

{notice}

{/if} + {#if editorOpen && canOperate} +
+

{editing ? 'Edit policy' : 'New policy'}

+
{ event.preventDefault(); void submit('preview'); }} oninput={() => preview = null} onchange={() => preview = null}> +
+
+ + + + + + + + +
+
+
+
+ {#if preview} +
+

Preview · {modeLabel(preview.policy.mode)}

+

Target: {preview.decision.desired} running lobbies. {preview.snapshot.keyedNames.length} saved lobby keys. {preview.snapshot.members.length} existing or pending lobbies.

+

{preview.decision.action === 'wait' ? 'Wait' : `${preview.decision.action === 'create' ? 'Create' : 'Remove'} ${preview.decision.server}`}: {preview.decision.reason}

+ {#if preview.policy.mode === 'create-only' || preview.policy.mode === 'create-destroy'}

Saving gives this policy exclusive control of the map's lobby count. Pause it before launching a competing collection or manual lifecycle action.

{/if} + {#if preview.snapshot.members.length}
    {#each preview.snapshot.members as member}
  • {member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}
    • {/each}
{/if} + {#each [...preview.blockers, ...preview.snapshot.blockers] as blocker}

{blocker}

{/each} + +
+ {/if} +
+ {/if} + {#if !report.policies.length}

No autoscaling policies. Add a policy to observe a map in dry run.

{/if} + {#each report.policies as view (view.policy.id)} +
+

{zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}

{modeLabel(view.policy.mode)}{#if canOperate}{/if}
+

{view.decision.occupied} occupied · {view.decision.empty} empty and ready · {view.decision.pending} pending · {view.decision.unknown} unknown or stopped

+

Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyedNames.length} saved keys

+

{view.policy.mode === 'dry-run' ? 'Hypothetical decision: ' : 'Current decision: '}{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}

+ {#if view.policy.mode === 'create-only' && view.decision.action === 'destroy'}

Removal is advisory in Create only mode.

{/if} +

Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}{view.policy.nextActionAt > Date.now() ? ` · Next action after ${$timeFormat.dateTime(new Date(view.policy.nextActionAt).toISOString())}` : ''}

+
Lobbies and evidence
    {#each view.snapshot.members as member}
  • {member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}
    {member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · Provisioning{/if}
    • {/each}
{#if !view.snapshot.members.length}

No registered or pending lobbies.

{/if}{#each view.snapshot.blockers as blocker}

{blocker}

{/each}
+
Recent decisions and measurements +

{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes

+

Up to 31 days of bounded observations; gaps are excluded. Dry run measures the actual fleet, not a simulated replacement lineup. Readiness does not certify browser visibility or successful joins.

+
    {#each view.history as item}
  1. {$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}
    {item.decision.reason}{#if item.operationId} · View operation{/if}
    2. {/each}
+
+
+ {/each} +
+ + diff --git a/src/routes/collections/+page.svelte b/src/routes/collections/+page.svelte index e6528db..aff5cf0 100644 --- a/src/routes/collections/+page.svelte +++ b/src/routes/collections/+page.svelte @@ -94,7 +94,7 @@ Collections | Electric Cherry
- {#snippet actions()}{#if canOperate}{/if}{/snippet} + {#snippet actions()}{#if canOperate}{/if}{/snippet} {#if canOperate} {/if} diff --git a/src/routes/metrics/+page.svelte b/src/routes/metrics/+page.svelte index edb91b1..8e39be6 100644 --- a/src/routes/metrics/+page.svelte +++ b/src/routes/metrics/+page.svelte @@ -108,7 +108,7 @@ Metrics · Electric Cherry
- + {#snippet actions()}{/snippet}