josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Support game-wide autoscaling rules with per-map targets

Browse Source
This commit is contained in:
josh
2026-10-06 23:34:16 -04:00
parent a61eccaed6
commit bc744e2840
11 changed files with 152 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/autoscaling/index.ts
+10 -4
View File
@@ -2,6 +2,7 @@ import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { zombieMaps, parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { cancelCreation } from '../provisioning/cancel-creation.js';
@@ -10,6 +11,7 @@ import { destroyServer } from '../provisioning/instances.js';
import { cancelRemoval } from '../provisioning/history.js';
import { autoscalingRemovalCapability, revokePolicyRemovals } from './authorization.js';
import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
@@ -29,7 +31,11 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`);
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
for (const map of zombieMaps(policy.game).filter(m => policy.mapSlug === '*' || m.slug === policy.mapSlug)) {
if (snapshot.keyedNames.filter(name => parseServerName(name)?.mapSlug === map.slug).length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities for ${map.friendly} before activating this policy.`);
}
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
@@ -52,7 +58,7 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke
if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
@@ -139,7 +145,7 @@ export async function runAutoscaling(): Promise<boolean> {
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
if (!result.ok) throw new Error(result.message);
@@ -155,7 +161,7 @@ export async function runAutoscaling(): Promise<boolean> {
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
})();
}
return submitted;
src/lib/server/autoscaling/ownership.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { getServer } from '../fleet/servers.js';
import { mapIdentity } from './snapshot.js';
export function scalingOwner(game: string, mapSlug: string): string | null {
return (db().prepare('SELECT id FROM autoscaling_policies WHERE game=? AND map_slug=? AND owns_map=1').get(game, mapSlug) as { id: string } | undefined)?.id ?? null;
return (db().prepare("SELECT id FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1").get(game, mapSlug, mapSlug) as { id: string } | undefined)?.id ?? null;
}
export function scalingOwnershipBlocker(server: string): string | null {
const identity = mapIdentity(getServer(server) ?? { name: server });
src/lib/server/autoscaling/planner.ts
+21 -1
View File
@@ -1,4 +1,5 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
@@ -14,13 +15,32 @@ export function emptyObservation(member: ScalingMember, previous: EmptyObservati
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches), keyedNames: snapshot.keyedNames.filter(matches)
}, now);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, occupied, empty, pending, unknown });
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
src/lib/server/autoscaling/snapshot.ts
+3 -3
View File
@@ -28,7 +28,7 @@ export function protectedWork(server: string, exceptOperation = ''): string | nu
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
@@ -54,12 +54,12 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const member of members) {