josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add autoscaling dry run and policy controls

Browse Source
This commit is contained in:
josh
2026-10-06 22:52:50 -04:00
parent ed4d8a4802
commit 36cf2b9c05
11 changed files with 317 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/autoscaling/index.ts
+97
View File
@@ -0,0 +1,97 @@
import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { scalingSnapshot } from './snapshot.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
export function previewScaling(input: unknown, id?: string, revision?: number): ScalingPreview {
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
if (id && !existing) throw new ScalingError('Policy not found.', 404);
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0,
nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
const snapshot = scalingSnapshot(policy);
const blockers = liveScaling(policy.mode) ? ['Live autoscaling is not available yet.'] : [];
const decision = planScaling(policy, snapshot, Date.now());
const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(),
members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
return { policy, snapshot, decision, token, blockers };
}
export function saveScaling(input: unknown, id?: string, revision?: number, token?: string): ScalingPolicy {
return db().transaction(() => {
const preview = previewScaling(input, id, revision);
if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409);
if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
return getScalingPolicy(nextId)!;
})();
}
export function setScalingGlobalPause(paused: boolean): void {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
}
export async function runAutoscaling(): Promise<void> {
const now = Date.now();
for (const policy of listScalingPolicies()) {
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
}
// Evidence includes provenance and timestamps but never keys or player rosters.
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
if (last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(randomUUID(), policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
})();
}
}
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1 AND operation_id IS NULL').run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
export function scalingReport(): ScalingReport {
const policies: ScalingView[] = listScalingPolicies().map(policy => {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id }));
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: false, removalAvailable: false };
}
src/lib/server/pipeline/worker.ts
+4 -2
View File
@@ -1,4 +1,5 @@
import { pruneHistory, pruneSampleHistory } from '../retention.js';
import { runAutoscaling, pruneAutoscaling } from '../autoscaling/index.js';
import { config } from '../config.js';
import { processProvisioning } from '../provisioning/worker.js';
import { sweep } from './sweep.js';
@@ -54,8 +55,9 @@ function reconcileRequesters(): void {
['campaign', 'Reconciling campaigns', sweep],
['beta', 'Reconciling beta tests', betaTick],
['maintenance', 'Checking maintenance and recovery', runMaintenance],
['autoscaling', 'Evaluating lobby autoscaling', runAutoscaling],
['game-files', 'Removing unused game files', cleanupControllerLibraries],
['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); }]
['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); }]
];
for (const [key, label, work] of requesters) {
if (requesterFlights.has(key)) continue;
@@ -65,7 +67,7 @@ function reconcileRequesters(): void {
}
const flight = trackActivity(key, label, work)
.catch(err => { console.error(`[worker] ${key} error:`, err); })
.then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); })
.then(async result => { if (key !== 'game-files' && key !== 'retention' && (key !== 'autoscaling' || result === true)) await tick(true); })
.finally(() => { requesterFlights.delete(key); });
requesterFlights.set(key, flight);
}