josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Support game-wide autoscaling rules with per-map targets

Browse Source
This commit is contained in:
josh
2026-10-06 23:34:16 -04:00
parent a61eccaed6
commit bc744e2840
11 changed files with 152 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/autoscaling.ts
+7 -4
View File
@@ -3,14 +3,17 @@ import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
export const autoscalingModes = ['off', 'dry-run', 'create-only', 'create-destroy'] as const;
export type AutoscalingMode = typeof autoscalingModes[number];
export interface ScalingSettings {
game: ZombieGame; mapSlug: string; mode: AutoscalingMode;
game: ZombieGame;
/** '*' applies the targets independently to every map. */
mapSlug: string;
mode: AutoscalingMode;
minimum: number; maximum: number; spares: number; emptyMinutes: number; cooldownMinutes: number;
}
export interface ScalingPolicy extends ScalingSettings {
id: string; revision: number; nextActionAt: number; pausedReason: string | null; updatedAt: string;
}
export interface ScalingMember {
name: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
name: string; mapSlug?: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
players: number | null; observedAt: number | null; source: string | null;
emptySince: number | null; blocker: string | null; operationId: string | null;
emptyStartedAt?: number | null;
@@ -20,7 +23,7 @@ export interface ScalingSnapshot {
}
export interface ScalingDecision {
action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null;
desired: number; occupied: number; empty: number; pending: number; unknown: number;
desired: number; deficit: number; occupied: number; empty: number; pending: number; unknown: number;
}
export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] }
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
@@ -38,7 +41,7 @@ export function scalingSettings(input: unknown): ScalingSettings {
if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.');
const value = input as Record<string, unknown>;
const { game, mapSlug } = value;
if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || zombieMapBySlug(game, mapSlug)?.slug !== mapSlug) throw new Error('Choose a supported game and map variant.');
if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || (mapSlug !== '*' && zombieMapBySlug(game, mapSlug)?.slug !== mapSlug)) throw new Error('Choose a supported game and map variant, or all maps.');
const mode = value.mode ?? 'off';
if (!autoscalingModes.includes(mode as AutoscalingMode)) throw new Error('Choose a valid autoscaling mode.');
const integer = (key: string, fallback: number | undefined, max: number) => {
src/lib/server/autoscaling/index.ts
+10 -4
View File
@@ -2,6 +2,7 @@ import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { zombieMaps, parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { cancelCreation } from '../provisioning/cancel-creation.js';
@@ -10,6 +11,7 @@ import { destroyServer } from '../provisioning/instances.js';
import { cancelRemoval } from '../provisioning/history.js';
import { autoscalingRemovalCapability, revokePolicyRemovals } from './authorization.js';
import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
@@ -29,7 +31,11 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`);
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
for (const map of zombieMaps(policy.game).filter(m => policy.mapSlug === '*' || m.slug === policy.mapSlug)) {
if (snapshot.keyedNames.filter(name => parseServerName(name)?.mapSlug === map.slug).length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities for ${map.friendly} before activating this policy.`);
}
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
@@ -52,7 +58,7 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke
if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
@@ -139,7 +145,7 @@ export async function runAutoscaling(): Promise<boolean> {
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
if (!result.ok) throw new Error(result.message);
@@ -155,7 +161,7 @@ export async function runAutoscaling(): Promise<boolean> {
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
})();
}
return submitted;
src/lib/server/autoscaling/ownership.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { getServer } from '../fleet/servers.js';
import { mapIdentity } from './snapshot.js';
export function scalingOwner(game: string, mapSlug: string): string | null {
return (db().prepare('SELECT id FROM autoscaling_policies WHERE game=? AND map_slug=? AND owns_map=1').get(game, mapSlug) as { id: string } | undefined)?.id ?? null;
return (db().prepare("SELECT id FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1").get(game, mapSlug, mapSlug) as { id: string } | undefined)?.id ?? null;
}
export function scalingOwnershipBlocker(server: string): string | null {
const identity = mapIdentity(getServer(server) ?? { name: server });
src/lib/server/autoscaling/planner.ts
+21 -1
View File
@@ -1,4 +1,5 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
@@ -14,13 +15,32 @@ export function emptyObservation(member: ScalingMember, previous: EmptyObservati
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches), keyedNames: snapshot.keyedNames.filter(matches)
}, now);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, occupied, empty, pending, unknown });
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
src/lib/server/autoscaling/snapshot.ts
+3 -3
View File
@@ -28,7 +28,7 @@ export function protectedWork(server: string, exceptOperation = ''): string | nu
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
@@ -54,12 +54,12 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const member of members) {