josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove autoscaling action cooldown

Browse Source
This commit is contained in:
josh
2026-10-07 15:51:52 -04:00
parent eadf97a428
commit a7a600b687
13 changed files with 80 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/autoscaling.ts
+3 -3
View File
@@ -7,10 +7,10 @@ export interface ScalingSettings {
/** '*' applies the targets independently to every map. */
mapSlug: string;
mode: AutoscalingMode;
minimum: number; maximum: number; spares: number; emptyMinutes: number; cooldownMinutes: number;
minimum: number; maximum: number; spares: number; emptyMinutes: number;
}
export interface ScalingPolicy extends ScalingSettings {
id: string; revision: number; nextActionAt: number; pausedReason: string | null; updatedAt: string;
id: string; revision: number; pausedReason: string | null; updatedAt: string;
}
export interface ScalingMember {
name: string; mapSlug?: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
@@ -52,5 +52,5 @@ export function scalingSettings(input: unknown): ScalingSettings {
const minimum = integer('minimum', 1, 20), maximum = integer('maximum', undefined, 20), spares = integer('spares', 1, 20);
if (minimum > maximum || spares > maximum) throw new Error('Minimum and spare target cannot exceed the maximum.');
return { game, mapSlug, mode: mode as AutoscalingMode, minimum, maximum, spares,
emptyMinutes: integer('emptyMinutes', 20, 1440), cooldownMinutes: integer('cooldownMinutes', 5, 1440) };
emptyMinutes: integer('emptyMinutes', 20, 1440) };
}
src/lib/server/autoscaling/admission.ts
+1 -1
View File
@@ -17,7 +17,7 @@ export function scalingAdmission(operationId: string): string | null {
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' };
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
src/lib/server/autoscaling/index.ts
+3 -4
View File
@@ -15,8 +15,8 @@ import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at });
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
@@ -27,7 +27,7 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0,
nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
@@ -167,7 +167,6 @@ export async function runAutoscaling(): Promise<boolean> {
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
submitted = true;
})();
} catch (error) {
src/lib/server/autoscaling/planner.ts
-1
View File
@@ -47,7 +47,6 @@ export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, no
const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
if (policy.nextActionAt > now) return result('wait', 'Action cooldown is active.');
if (occupied + empty < desired) {
if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
src/lib/server/autoscaling/schema.ts
+9 -3
View File
@@ -5,7 +5,7 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
CREATE TABLE IF NOT EXISTS autoscaling_policies (
id TEXT PRIMARY KEY, game TEXT NOT NULL, map_slug TEXT NOT NULL,
revision INTEGER NOT NULL DEFAULT 1, settings TEXT NOT NULL,
mode TEXT NOT NULL DEFAULT 'off', next_action_at INTEGER NOT NULL DEFAULT 0,
mode TEXT NOT NULL DEFAULT 'off',
paused_reason TEXT, owns_map INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL,
UNIQUE(game,map_slug)
);
@@ -34,9 +34,15 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
);
CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at);
`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.transaction(() => {
const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[];
if (columns.some(column => column.name === 'next_action_at')) {
handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at');
}
handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes')
WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=?
WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now());
src/routes/autoscaling/+page.svelte
+3 -4
View File
@@ -12,7 +12,7 @@
let snapshot = $state<ScalingReport | null>(null);
const report = $derived(snapshot ?? data.autoscaling);
const canOperate = $derived(page.data.user?.role === 'admin' || page.data.user?.role === 'operator');
const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'off', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20, cooldownMinutes: 5 });
const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'off', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20 });
let settings = $state<ScalingSettings>(defaults());
let editing = $state<ScalingPolicy | null>(null);
let editorOpen = $state(false);
@@ -92,9 +92,8 @@
<label>Maximum lobbies<input type="number" min="1" max="20" required bind:value={settings.maximum} /></label>
<label>Empty spare target<input type="number" min="1" max="20" required bind:value={settings.spares} /></label>
<label>Empty duration in minutes<input type="number" min="1" max="1440" required bind:value={settings.emptyMinutes} /></label>
<label>Action cooldown in minutes<input type="number" min="1" max="1440" required bind:value={settings.cooldownMinutes} /></label>
</div>
{#if settings.mapSlug === '*'}<p>Targets apply separately to every map, including all variants. All maps share this game’s key pool. Actions share one cooldown and run one at a time across the game.</p>{/if}
{#if settings.mapSlug === '*'}<p>Targets apply separately to every map, including all variants. All maps share this game’s key pool. Actions run one at a time across the game.</p>{/if}
<p>Each new lobby needs an available key from the game pool. Creation waits when the pool is exhausted; keys return after confirmed removal.</p>
<div class="actions"><Button type="submit" disabled={busy}>Preview changes</Button><Button onclick={() => editorOpen = false}>Close editor</Button></div>
</fieldset>
@@ -120,7 +119,7 @@
{#if view.unsettled.length}<div class="banner"><strong>Work is still settling.</strong><p>Pause this policy and settle its outstanding work before deleting it.</p>{#each view.unsettled as operation}<p><a href={`/provisioning/${operation.id}`}>{operation.action} · {operation.state}</a> — {operation.stage}</p>{/each}{#if view.ownsMap}<p>This policy retains ownership of its scope until its outstanding work settles.</p>{/if}</div>{/if}
<p>{view.policy.mapSlug === '*' ? 'Per map: ' : ''}Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyPool.available} / {view.snapshot.keyPool.total} game keys available</p>
<p><strong>Current decision: </strong>{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}</p>
<p class="muted">Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}{view.policy.nextActionAt > Date.now() ? ` · Next action after ${$timeFormat.dateTime(new Date(view.policy.nextActionAt).toISOString())}` : ''}</p>
<p class="muted">Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}</p>
<details><summary>Lobbies and evidence</summary><ul>{#each view.snapshot.members as member}<li><a href={`/servers/${encodeURIComponent(member.name)}`}>{member.name}</a> · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}<br />{member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · <a href={`/provisioning/${member.operationId}`}>Provisioning</a>{/if}</li>{/each}</ul>{#if !view.snapshot.members.length}<p>No registered or pending lobbies.</p>{/if}{#each view.snapshot.blockers as blocker}<p>{blocker}</p>{/each}</details>
<details><summary>Recent decisions and measurements</summary>
<p>{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes</p>