Add autoscaling dry run and policy controls
This commit is contained in:
@@ -21,6 +21,13 @@ export interface ScalingDecision {
|
||||
action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null;
|
||||
desired: number; occupied: number; empty: number; pending: number; unknown: number;
|
||||
}
|
||||
export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] }
|
||||
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null }
|
||||
export interface ScalingView extends ScalingPreview {
|
||||
history: ScalingHistory[]; lastEvaluatedAt: number | null;
|
||||
totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
|
||||
}
|
||||
export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
|
||||
export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy';
|
||||
|
||||
export function scalingSettings(input: unknown): ScalingSettings {
|
||||
|
||||
@@ -7,6 +7,7 @@ export const navigationGroups: NavigationGroup[] = [
|
||||
{ id: 'fleet', label: 'Fleet', items: [
|
||||
{ label: 'Overview', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') },
|
||||
{ label: 'Collections', href: '/collections' },
|
||||
{ label: 'Autoscaling', href: '/autoscaling' },
|
||||
{ label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) },
|
||||
{ label: 'Capacity', href: '/capacity' }
|
||||
] },
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
|
||||
import { emptyObservation, planScaling } from './planner.js';
|
||||
import { scalingSnapshot } from './snapshot.js';
|
||||
|
||||
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
|
||||
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
|
||||
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at });
|
||||
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
|
||||
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
|
||||
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
|
||||
|
||||
export function previewScaling(input: unknown, id?: string, revision?: number): ScalingPreview {
|
||||
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
|
||||
if (id && !existing) throw new ScalingError('Policy not found.', 404);
|
||||
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
|
||||
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
|
||||
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0,
|
||||
nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
|
||||
const snapshot = scalingSnapshot(policy);
|
||||
const blockers = liveScaling(policy.mode) ? ['Live autoscaling is not available yet.'] : [];
|
||||
const decision = planScaling(policy, snapshot, Date.now());
|
||||
const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(),
|
||||
members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
|
||||
return { policy, snapshot, decision, token, blockers };
|
||||
}
|
||||
|
||||
export function saveScaling(input: unknown, id?: string, revision?: number, token?: string): ScalingPolicy {
|
||||
return db().transaction(() => {
|
||||
const preview = previewScaling(input, id, revision);
|
||||
if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409);
|
||||
if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
|
||||
const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
|
||||
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
|
||||
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
|
||||
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
|
||||
// An edit starts a new observation interval; earlier inactivity is not authorization.
|
||||
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
|
||||
return getScalingPolicy(nextId)!;
|
||||
})();
|
||||
}
|
||||
|
||||
export function setScalingGlobalPause(paused: boolean): void {
|
||||
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
|
||||
}
|
||||
|
||||
export async function runAutoscaling(): Promise<void> {
|
||||
const now = Date.now();
|
||||
for (const policy of listScalingPolicies()) {
|
||||
if (policy.mode === 'off') continue;
|
||||
db().transaction(() => {
|
||||
const snapshot = scalingSnapshot(policy, now);
|
||||
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
|
||||
const decision = planScaling(policy, snapshot, now);
|
||||
for (const member of snapshot.members) {
|
||||
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
|
||||
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
|
||||
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
|
||||
}
|
||||
// Evidence includes provenance and timestamps but never keys or player rosters.
|
||||
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
|
||||
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
|
||||
if (last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
|
||||
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
|
||||
.run(randomUUID(), policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
|
||||
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
|
||||
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
export function pruneAutoscaling(now = Date.now()): void {
|
||||
const cutoff = now - 31 * 86400_000;
|
||||
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1 AND operation_id IS NULL').run(cutoff);
|
||||
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
|
||||
}
|
||||
|
||||
export function scalingReport(): ScalingReport {
|
||||
const policies: ScalingView[] = listScalingPolicies().map(policy => {
|
||||
const snapshot = scalingSnapshot(policy);
|
||||
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
|
||||
const decision = planScaling(policy, snapshot, Date.now());
|
||||
const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as
|
||||
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[])
|
||||
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id }));
|
||||
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
|
||||
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
|
||||
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
|
||||
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
|
||||
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
|
||||
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
|
||||
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
|
||||
});
|
||||
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: false, removalAvailable: false };
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { pruneHistory, pruneSampleHistory } from '../retention.js';
|
||||
import { runAutoscaling, pruneAutoscaling } from '../autoscaling/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { processProvisioning } from '../provisioning/worker.js';
|
||||
import { sweep } from './sweep.js';
|
||||
@@ -54,8 +55,9 @@ function reconcileRequesters(): void {
|
||||
['campaign', 'Reconciling campaigns', sweep],
|
||||
['beta', 'Reconciling beta tests', betaTick],
|
||||
['maintenance', 'Checking maintenance and recovery', runMaintenance],
|
||||
['autoscaling', 'Evaluating lobby autoscaling', runAutoscaling],
|
||||
['game-files', 'Removing unused game files', cleanupControllerLibraries],
|
||||
['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); }]
|
||||
['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); }]
|
||||
];
|
||||
for (const [key, label, work] of requesters) {
|
||||
if (requesterFlights.has(key)) continue;
|
||||
@@ -65,7 +67,7 @@ function reconcileRequesters(): void {
|
||||
}
|
||||
const flight = trackActivity(key, label, work)
|
||||
.catch(err => { console.error(`[worker] ${key} error:`, err); })
|
||||
.then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); })
|
||||
.then(async result => { if (key !== 'game-files' && key !== 'retention' && (key !== 'autoscaling' || result === true)) await tick(true); })
|
||||
.finally(() => { requesterFlights.delete(key); });
|
||||
requesterFlights.set(key, flight);
|
||||
}
|
||||
|
||||
@@ -143,6 +143,7 @@
|
||||
<section class="page">
|
||||
<SectionHeader title="Fleet" subtitle="Live health, players, and server operations.">
|
||||
{#snippet actions()}
|
||||
<Button href="/autoscaling" variant="ghost" size="sm">Autoscaling</Button>
|
||||
<Button variant="ghost" size="sm" loading={$fleetRefreshing} onclick={() => void refreshFleet()}>Refresh</Button>
|
||||
{#if canOperate}<Button variant="primary" onclick={() => launch()}>Create server</Button>{/if}
|
||||
{/snippet}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { previewScaling, saveScaling, scalingReport, setScalingGlobalPause, ScalingError } from '$lib/server/autoscaling/index.js';
|
||||
|
||||
export function GET() { return json(scalingReport()); }
|
||||
export async function POST({ request }: { request: Request }) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new ScalingError('Provide a policy action.');
|
||||
if (body.id !== undefined && typeof body.id !== 'string') throw new ScalingError('Invalid policy identity.');
|
||||
if (body.action === 'preview') return json(previewScaling(body.settings, body.id, body.revision));
|
||||
if (body.action === 'save') return json({ policy: saveScaling(body.settings, body.id, body.revision, body.token) });
|
||||
if (body.action === 'global-pause' && typeof body.paused === 'boolean') { setScalingGlobalPause(body.paused); return json({ ok: true }); }
|
||||
throw new ScalingError('Choose a supported policy action.');
|
||||
} catch (error) { return json({ error: (error as Error).message }, { status: error instanceof ScalingError ? error.status : 400 }); }
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import { scalingReport } from '$lib/server/autoscaling/index.js';
|
||||
export function load() { return { autoscaling: scalingReport() }; }
|
||||
@@ -0,0 +1,133 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/state';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { timeFormat } from '$ui/stores/time-zone.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import { zombieMaps, zombieMapBySlug } from '$lib/map-aliases.js';
|
||||
import type { ScalingSettings, ScalingPolicy, ScalingPreview, ScalingReport } from '$lib/autoscaling.js';
|
||||
let { data }: { data: import('./$types').PageData } = $props();
|
||||
let snapshot = $state<ScalingReport | null>(null);
|
||||
const report = $derived(snapshot ?? data.autoscaling);
|
||||
const canOperate = $derived(page.data.user?.role === 'admin' || page.data.user?.role === 'operator');
|
||||
const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'dry-run', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20, cooldownMinutes: 5 });
|
||||
let settings = $state<ScalingSettings>(defaults());
|
||||
let editing = $state<ScalingPolicy | null>(null);
|
||||
let editorOpen = $state(false);
|
||||
let preview = $state<ScalingPreview | null>(null);
|
||||
let busy = $state(false), actionError = $state(''), refreshError = $state(''), notice = $state('');
|
||||
let generation = 0;
|
||||
const modeLabel = (mode: string) => ({ off: 'Off', 'dry-run': 'Dry run', 'create-only': 'Create only', 'create-destroy': 'Create and destroy' })[mode] ?? mode;
|
||||
async function refresh(signal?: AbortSignal) {
|
||||
const version = ++generation;
|
||||
try {
|
||||
const response = await fetch('/api/autoscaling', { signal });
|
||||
if (!response.ok) throw new Error('Could not refresh policies. The last successful report is shown.');
|
||||
const result = await response.json();
|
||||
if (version === generation && !signal?.aborted) { snapshot = result; refreshError = ''; }
|
||||
} catch (error) { if (version === generation && !signal?.aborted) refreshError = (error as Error).message; }
|
||||
}
|
||||
onMount(() => pollVisible(refresh, 15_000, false));
|
||||
function edit(policy: ScalingPolicy | null) {
|
||||
editing = policy; settings = policy ? { ...policy } : defaults();
|
||||
preview = null; actionError = ''; notice = ''; editorOpen = true;
|
||||
}
|
||||
async function post(body: object) {
|
||||
const response = await fetch('/api/autoscaling', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error ?? 'Policy action failed.');
|
||||
return result;
|
||||
}
|
||||
async function submit(action: 'preview' | 'save') {
|
||||
busy = true; actionError = ''; notice = ''; ++generation;
|
||||
try {
|
||||
const result = await post({ action, settings, id: editing?.id, revision: editing?.revision, token: preview?.token });
|
||||
if (action === 'preview') preview = result;
|
||||
else { notice = 'Policy saved. Evaluation runs every 15 seconds.'; editorOpen = false; preview = null; await refresh(); }
|
||||
} catch (error) { actionError = (error as Error).message; preview = null; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
async function pauseAll() {
|
||||
busy = true; actionError = ''; ++generation;
|
||||
try { await post({ action: 'global-pause', paused: !report.globalPaused }); notice = report.globalPaused ? 'Autoscaling resumed.' : 'Autoscaling paused. Review any unsettled operations below.'; await refresh(); }
|
||||
catch (error) { actionError = (error as Error).message; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Autoscaling | Electric Cherry</title></svelte:head>
|
||||
<div class="page">
|
||||
<SectionHeader title="Autoscaling" subtitle="Keep empty lobbies available for new groups, within the limits you choose.">
|
||||
{#snippet actions()}<Button onclick={() => refresh()}>Refresh</Button>{#if canOperate}<Button disabled={busy} onclick={pauseAll}>{report.globalPaused ? 'Resume autoscaling' : 'Pause all'}</Button><Button variant="primary" onclick={() => edit(null)}>Add policy</Button>{/if}{/snippet}
|
||||
</SectionHeader>
|
||||
<p class="context"><a href="/">Fleet</a> · <a href="/collections">Collections</a> · <a href="/metrics">Metrics</a></p>
|
||||
{#if report.globalPaused}<p class="banner">All autoscaling decisions are paused. Delivered operations may still need to settle.</p>{/if}
|
||||
<p>Start with a two-week dry run. Dry run records hypothetical actions and reserves no lobby or host capacity.</p>
|
||||
{#if refreshError}<p role="alert">{refreshError}</p>{/if}
|
||||
{#if actionError}<p role="alert">{actionError}</p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if editorOpen && canOperate}
|
||||
<section class="panel" aria-labelledby="policy-editor">
|
||||
<h2 id="policy-editor">{editing ? 'Edit policy' : 'New policy'}</h2>
|
||||
<form onsubmit={event => { event.preventDefault(); void submit('preview'); }} oninput={() => preview = null} onchange={() => preview = null}>
|
||||
<fieldset disabled={busy}>
|
||||
<div class="fields">
|
||||
<label>Game<select bind:value={settings.game} disabled={!!editing} onchange={() => settings.mapSlug = zombieMaps(settings.game)[0].slug}><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label>
|
||||
<label>Map variant<select bind:value={settings.mapSlug} disabled={!!editing}>{#each zombieMaps(settings.game) as map}<option value={map.slug}>{map.friendly}</option>{/each}</select></label>
|
||||
<label>Mode<select bind:value={settings.mode}><option value="off">Off</option><option value="dry-run">Dry run</option>{#if report.liveAvailable}<option value="create-only">Create only</option>{/if}{#if report.removalAvailable}<option value="create-destroy">Create and destroy</option>{/if}</select></label>
|
||||
<label>Minimum running lobbies<input type="number" min="1" max="20" required bind:value={settings.minimum} /></label>
|
||||
<label>Maximum lobbies<input type="number" min="1" max="20" required bind:value={settings.maximum} /></label>
|
||||
<label>Empty spare target<input type="number" min="1" max="20" required bind:value={settings.spares} /></label>
|
||||
<label>Empty duration in minutes<input type="number" min="1" max="1440" required bind:value={settings.emptyMinutes} /></label>
|
||||
<label>Action cooldown in minutes<input type="number" min="1" max="1440" required bind:value={settings.cooldownMinutes} /></label>
|
||||
</div>
|
||||
<div class="actions"><Button type="submit" disabled={busy}>Preview changes</Button><Button onclick={() => editorOpen = false}>Close editor</Button></div>
|
||||
</fieldset>
|
||||
</form>
|
||||
{#if preview}
|
||||
<div class="preview" aria-live="polite">
|
||||
<h3>Preview · {modeLabel(preview.policy.mode)}</h3>
|
||||
<p>Target: {preview.decision.desired} running lobbies. {preview.snapshot.keyedNames.length} saved lobby keys. {preview.snapshot.members.length} existing or pending lobbies.</p>
|
||||
<p>{preview.decision.action === 'wait' ? 'Wait' : `${preview.decision.action === 'create' ? 'Create' : 'Remove'} ${preview.decision.server}`}: {preview.decision.reason}</p>
|
||||
{#if preview.policy.mode === 'create-only' || preview.policy.mode === 'create-destroy'}<p>Saving gives this policy exclusive control of the map's lobby count. Pause it before launching a competing collection or manual lifecycle action.</p>{/if}
|
||||
{#if preview.snapshot.members.length}<ul>{#each preview.snapshot.members as member}<li>{member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}</li>{/each}</ul>{/if}
|
||||
{#each [...preview.blockers, ...preview.snapshot.blockers] as blocker}<p>{blocker}</p>{/each}
|
||||
<Button variant="primary" disabled={busy || preview.blockers.length > 0} onclick={() => submit('save')}>Save reviewed policy</Button>
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
{/if}
|
||||
{#if !report.policies.length}<p class="panel">No autoscaling policies. Add a policy to observe a map in dry run.</p>{/if}
|
||||
{#each report.policies as view (view.policy.id)}
|
||||
<section class="panel" id={view.policy.id} aria-labelledby={`title-${view.policy.id}`}>
|
||||
<div class="heading"><h2 id={`title-${view.policy.id}`}>{zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}</h2><span>{modeLabel(view.policy.mode)}</span>{#if canOperate}<Button onclick={() => edit(view.policy)}>Edit or pause</Button>{/if}</div>
|
||||
<p class="counts">{view.decision.occupied} occupied · {view.decision.empty} empty and ready · {view.decision.pending} pending · {view.decision.unknown} unknown or stopped</p>
|
||||
<p>Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyedNames.length} saved keys</p>
|
||||
<p><strong>{view.policy.mode === 'dry-run' ? 'Hypothetical decision: ' : 'Current decision: '}</strong>{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}</p>
|
||||
{#if view.policy.mode === 'create-only' && view.decision.action === 'destroy'}<p>Removal is advisory in Create only mode.</p>{/if}
|
||||
<p class="muted">Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}{view.policy.nextActionAt > Date.now() ? ` · Next action after ${$timeFormat.dateTime(new Date(view.policy.nextActionAt).toISOString())}` : ''}</p>
|
||||
<details><summary>Lobbies and evidence</summary><ul>{#each view.snapshot.members as member}<li><a href={`/servers/${encodeURIComponent(member.name)}`}>{member.name}</a> · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}<br />{member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · <a href={`/provisioning/${member.operationId}`}>Provisioning</a>{/if}</li>{/each}</ul>{#if !view.snapshot.members.length}<p>No registered or pending lobbies.</p>{/if}{#each view.snapshot.blockers as blocker}<p>{blocker}</p>{/each}</details>
|
||||
<details><summary>Recent decisions and measurements</summary>
|
||||
<p>{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes</p>
|
||||
<p class="muted">Up to 31 days of bounded observations; gaps are excluded. Dry run measures the actual fleet, not a simulated replacement lineup. Readiness does not certify browser visibility or successful joins.</p>
|
||||
<ol>{#each view.history as item}<li>{$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}<br />{item.decision.reason}{#if item.operationId} · <a href={`/provisioning/${item.operationId}`}>View operation</a>{/if}</li>{/each}</ol>
|
||||
</details>
|
||||
</section>
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page { max-width: 1200px; margin: 0 auto; padding: 24px; }
|
||||
.panel { margin: 20px 0; padding: 20px; border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); background: var(--surface); overflow-wrap: anywhere; }
|
||||
.heading, .actions { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
|
||||
h2 { font-size: var(--text-lg); margin: 0; } h3 { font-size: var(--text-md); }
|
||||
.heading h2 { flex: 1; } p, li { line-height: 1.6; } .muted, .context { color: var(--text-secondary); font-size: var(--text-sm); }
|
||||
.fields { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 220px), 1fr)); gap: 16px; margin-bottom: 20px; }
|
||||
label { display: flex; flex-direction: column; gap: 8px; font-size: var(--text-sm); }
|
||||
input, select { padding: 10px; min-width: 0; color: var(--text-primary); background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); font: inherit; }
|
||||
fieldset { border: 0; padding: 0; margin: 16px 0; min-width: 0; }
|
||||
.preview, .banner { padding: 16px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); }
|
||||
details { margin-top: 16px; } summary { cursor: pointer; } li { margin: 10px 0; }
|
||||
@media (max-width: 500px) { .page { padding: 16px; } .panel { padding: 16px; } }
|
||||
</style>
|
||||
@@ -94,7 +94,7 @@
|
||||
<svelte:head><title>Collections | Electric Cherry</title></svelte:head>
|
||||
<div class="page">
|
||||
<SectionHeader title="Collections" subtitle="Save your lineup, preview fleet changes, and follow launch progress and history.">
|
||||
{#snippet actions()}{#if canOperate}<Button variant="primary" onclick={() => { editingBlueprint = null; builderOpen = true; }}>Create collection</Button>{/if}{/snippet}
|
||||
{#snippet actions()}<Button href="/autoscaling">Autoscaling</Button>{#if canOperate}<Button variant="primary" onclick={() => { editingBlueprint = null; builderOpen = true; }}>Create collection</Button>{/if}{/snippet}
|
||||
</SectionHeader>
|
||||
{#if canOperate}<BlueprintBuilder bind:open={builderOpen} blueprint={editingBlueprint} onsaved={blueprintSaved} />
|
||||
<DeployCollectionDialog bind:open={collectionOpen} {blueprintId} />{/if}
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
<svelte:head><title>Metrics · Electric Cherry</title></svelte:head>
|
||||
|
||||
<div class="page">
|
||||
<SectionHeader title="Metrics" />
|
||||
<SectionHeader title="Metrics">{#snippet actions()}<Button href="/autoscaling">Autoscaling</Button>{/snippet}</SectionHeader>
|
||||
<div class="toolbar">
|
||||
<nav aria-label="Metrics time range">
|
||||
{#each [1, 7, 30] as days}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
|
||||
const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' };
|
||||
|
||||
it('persists deduplicated dry-run decisions across restart without provisioning or reservations', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
||||
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_3', 'secret');
|
||||
const preview = scaling.previewScaling(settings);
|
||||
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
|
||||
await scaling.runAutoscaling(); await scaling.runAutoscaling();
|
||||
closeDb();
|
||||
await scaling.runAutoscaling();
|
||||
const view = scaling.scalingReport().policies[0];
|
||||
expect(view.policy.id).toBe(saved.id);
|
||||
expect(view.history).toHaveLength(1);
|
||||
expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_3' });
|
||||
expect(JSON.stringify(view)).not.toContain('secret');
|
||||
for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 });
|
||||
});
|
||||
|
||||
it('rejects duplicate policies, stale edits and changed previews', async () => {
|
||||
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
||||
const preview = scaling.previewScaling(settings);
|
||||
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
|
||||
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists');
|
||||
expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
|
||||
expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
|
||||
expect(() => scaling.saveScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision).token)).toThrow('not available');
|
||||
});
|
||||
|
||||
it('pauses evaluations and prunes expired observations without deleting current history', async () => {
|
||||
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
||||
scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
|
||||
await scaling.runAutoscaling();
|
||||
scaling.setScalingGlobalPause(true); await scaling.runAutoscaling();
|
||||
expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const id = scaling.scalingReport().policies[0].history[0].id;
|
||||
db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id);
|
||||
scaling.pruneAutoscaling();
|
||||
expect(scaling.scalingReport().policies[0].history).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('uses fleet authorization for page, reads, and mutations', () => {
|
||||
expect(routePermission('/autoscaling', 'GET')).toBe('viewer');
|
||||
expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer');
|
||||
expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false);
|
||||
expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true);
|
||||
});
|
||||
Reference in New Issue
Block a user