Add safe manual instance rebalancing

This commit is contained in:
2026-10-08 12:37:56 -04:00
parent 26de43639f
commit 012cd51a66
45 changed files with 2006 additions and 154 deletions
@@ -0,0 +1,160 @@
<script lang="ts">
import { onMount, untrack } from 'svelte';
import type { RebalancePreview, RebalanceReport, RebalanceRun, RebalanceSettings } from '$lib/rebalancing.js';
import { pollVisible } from '$ui/poll.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { rebalanceRunLabels } from '$ui/rebalancing-presentation.js';
import Button from '$ui/components/primitives/Button.svelte';
import Freshness from '$ui/components/Freshness.svelte';
import RebalancePreviewView from './RebalancePreviewView.svelte';
import RebalanceRunView from './RebalanceRunView.svelte';
let { canManage, preferredHost = null, preferredRun = null }: {
canManage: boolean; preferredHost?: string | null; preferredRun?: string | null;
} = $props();
let report = $state<RebalanceReport | null>(null);
let selectedHosts = $state<string[]>([]);
let game = $state<RebalanceSettings['game']>('all');
let maxMoves = $state<number | undefined>(5);
let preview = $state<RebalancePreview | null>(null);
let selectedRun = $state(untrack(() => preferredRun));
let busy = $state<'preview' | 'start' | 'stop' | null>(null);
let loading = $state(false);
let refreshError = $state('');
let actionError = $state('');
let notice = $state('');
let now = $state(untrack(() => Date.now()));
let checkedAt = $state<number | null>(null);
let initialized = false;
let sequence = 0;
let loadRequest: AbortController | null = null;
let actionRequest: AbortController | null = null;
const activeRun = $derived(report?.runs.find(run => run.state === 'running' || run.state === 'stopping') ?? null);
const run = $derived(report?.runs.find(item => item.id === selectedRun) ?? activeRun ?? report?.runs[0] ?? null);
const validSettings = $derived(selectedHosts.length >= 2 && selectedHosts.length <= 100 && Number.isInteger(maxMoves) && maxMoves! >= 1 && maxMoves! <= 20);
const expired = $derived(!!preview && Date.parse(preview.expiresAt) <= now);
function editSettings() { preview = null; actionError = ''; notice = ''; }
function toggleHost(id: string, checked: boolean) {
selectedHosts = checked ? [...selectedHosts, id] : selectedHosts.filter(value => value !== id);
editSettings();
}
function cancelLoad() { sequence++; loadRequest?.abort(); loadRequest = null; loading = false; }
async function load(signal?: AbortSignal) {
if (busy || loading) return;
const current = ++sequence;
const controller = new AbortController();
loadRequest = controller; loading = true;
try {
const response = await fetch('/api/rebalancing', { signal: AbortSignal.any([controller.signal, AbortSignal.timeout(20_000), ...(signal ? [signal] : [])]) });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not refresh rebalancing.');
if (current !== sequence) return;
report = body as RebalanceReport; refreshError = ''; checkedAt = Date.now(); now = Date.now();
if (!initialized) {
selectedHosts = preferredHost ? report.hosts.filter(host => host.id === preferredHost && host.eligible).map(host => host.id)
: report.hosts.filter(host => host.eligible).map(host => host.id).slice(0, 100);
initialized = true;
} else {
const eligible = new Set(report.hosts.filter(host => host.eligible).map(host => host.id));
const available = selectedHosts.filter(id => eligible.has(id));
if (available.length !== selectedHosts.length) {
selectedHosts = available; preview = null;
notice = 'Host availability changed. Unavailable hosts were removed from your selection; preview again before starting.';
}
}
} catch (cause) {
if (current === sequence && !controller.signal.aborted && !signal?.aborted) refreshError = (cause as Error).message;
return false;
} finally { if (current === sequence) { loadRequest = null; loading = false; } }
}
async function act(action: 'preview' | 'start' | 'stop') {
if (busy || !canManage) return;
const payload = action === 'preview' ? { action, settings: { hostIds: selectedHosts, game, maxMoves } }
: action === 'start' ? { action, previewId: preview?.id } : { action, id: run?.id };
cancelLoad(); busy = action; actionError = ''; notice = '';
if (action === 'preview') preview = null;
const controller = new AbortController(); actionRequest = controller;
try {
const response = await fetch('/api/rebalancing', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload), signal: AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]) });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Rebalance request failed.');
if (action === 'preview') preview = body.preview;
else {
const updated = body.run as RebalanceRun;
if (report) report = { ...report, runs: [updated, ...report.runs.filter(item => item.id !== updated.id)] };
selectedRun = updated.id; preview = null;
notice = action === 'start' ? 'Rebalance started. Moves run one at a time.' : 'Unstarted moves have been withdrawn. Any move already in progress will finish.';
}
now = Date.now();
} catch (cause) {
if (!controller.signal.aborted) actionError = `${(cause as Error).message}${action === 'preview' ? '' : ' Refresh the run status before retrying an unconfirmed request.'}`;
} finally {
if (!controller.signal.aborted) { busy = null; actionRequest = null; await load(); }
}
}
onMount(() => {
const stop = pollVisible(load, 10_000);
const timer = setInterval(() => now = Date.now(), 5_000);
return () => { stop(); clearInterval(timer); cancelLoad(); actionRequest?.abort(); };
});
</script>
<section class="panel" aria-labelledby="rebalance-title">
<div class="heading"><div><h2 id="rebalance-title">Rebalance instances</h2><p>Spread existing instances across hosts according to CPU and memory use relative to each host’s capacity.</p></div>
{#if checkedAt && !refreshError}<Freshness updatedAt={checkedAt} intervalLabel="Updates every 10 seconds" />{/if}
</div>
<p class="help">Only healthy, empty lobbies can move. Each move rebuilds the lobby on another host, causing a temporary outage; its address or port may change. The server identity, map, key, and recorded deployed files are retained. Local mutable files are not copied.</p>
{#if refreshError}<div class="error" role="alert"><p>{refreshError}{report ? ' Showing the last loaded rebalancing status.' : ''}</p><Button size="sm" disabled={loading || !!busy} onclick={() => void load()}>Retry</Button></div>{/if}
{#if actionError}<p class="error" role="alert">{actionError}</p>{/if}
{#if notice}<p class="notice" role="status">{notice}</p>{/if}
{#if !report && loading}<p role="status">Loading hosts and rebalance runs…</p>{/if}
{#if report}
{#if canManage}
<form onsubmit={event => { event.preventDefault(); void act('preview'); }}>
<fieldset disabled={!!busy || !!activeRun}>
<legend>Participating hosts</legend>
<p class="help">Choose at least two hosts. Existing lobbies may move between any selected hosts.</p>
{#if preferredHost && !report.hosts.some(host => host.id === preferredHost && host.eligible)}<p class="help">The host opened from the shortcut is currently unavailable. Choose eligible hosts below.</p>{/if}
<div class="host-selection">{#each report.hosts as host (host.id)}
<label class:unavailable={!host.eligible}><input type="checkbox" checked={selectedHosts.includes(host.id)} disabled={!host.eligible} onchange={event => toggleHost(host.id, event.currentTarget.checked)} />
<span><strong>{host.name}</strong>{#if host.reason}<small>{host.reason}</small>{/if}</span></label>
{/each}</div>
<div class="settings"><label>Game<select bind:value={game} onchange={editSettings}><option value="all">All games</option><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label>
<label>Maximum moves<input type="number" min="1" max="20" step="1" required bind:value={maxMoves} oninput={editSettings} /></label>
<Button type="submit" disabled={!validSettings || !!busy || !!activeRun || !!refreshError} loading={busy === 'preview'}>Preview rebalance</Button>
</div>
</fieldset>
</form>
{#if activeRun}<p class="help">A rebalance is {activeRun.state === 'stopping' ? 'finishing its current move' : 'already in progress'}. A new run can start after it finishes.{#if activeRun.id !== run?.id} <button class="text-button" onclick={() => selectedRun = activeRun.id}>View active run</button>{/if}</p>{/if}
{:else}<p class="help">Operators and administrators can preview, start, and stop rebalance runs.</p>{/if}
{#if preview}<RebalancePreviewView {preview} {expired} disabled={!!busy || !!activeRun || !!refreshError} starting={busy === 'start'} onstart={() => void act('start')} />{/if}
<section class="runs" aria-labelledby="rebalance-runs-title">
<div class="heading"><h3 id="rebalance-runs-title">Rebalance runs</h3>{#if report.runs.length}<label class="run-picker"><span class="sr-only">Select rebalance run</span><select value={run?.id ?? ''} onchange={event => selectedRun = event.currentTarget.value}>{#each report.runs as item (item.id)}<option value={item.id}>{$timeFormat.dateTime(item.createdAt)} · {rebalanceRunLabels[item.state]}</option>{/each}</select></label>{/if}</div>
{#if preferredRun && !report.runs.some(item => item.id === preferredRun)}<p class="help">The linked run is not in the recent history.</p>{/if}
{#if run}<RebalanceRunView {run} {canManage} busy={!!busy} stopping={busy === 'stop'} onstop={() => void act('stop')} />
{:else}<p class="help">No rebalance runs yet.</p>{/if}
</section>
{/if}
</section>
<style>
.panel { display:grid; gap:var(--space-5); padding:24px; border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); min-width:0; }
h2,h3,p { margin:0; } h2 { font-size:var(--text-lg); } h3 { font-size:var(--text-md); } p { font-size:var(--text-sm); line-height:1.6; } .heading>div>p { margin-top:8px; color:var(--text-secondary); }
.heading,.settings { display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:12px; }
.help { font-size:var(--text-xs); color:var(--text-secondary); line-height:1.6; }
fieldset { border:0; padding:0; margin:0; display:grid; gap:12px; min-width:0; } legend { font-size:var(--text-sm); font-weight:600; padding:0 0 8px; }
.host-selection { display:grid; grid-template-columns:repeat(auto-fit,minmax(min(100%,220px),1fr)); gap:8px; }
.host-selection label { display:flex; align-items:flex-start; gap:10px; padding:12px; border:1px solid var(--border-subtle); border-radius:var(--radius-sm); cursor:pointer; }
.host-selection input { margin:3px 0 0; accent-color:var(--accent-primary); }
.host-selection span { display:grid; gap:4px; min-width:0; } .host-selection strong { font-size:var(--text-sm); font-weight:500; overflow-wrap:anywhere; }
.host-selection small { font-size:var(--text-xs); color:var(--text-secondary); line-height:1.5; } .host-selection .unavailable { opacity:0.65; cursor:default; }
.settings { justify-content:flex-start; align-items:end; } .settings label { display:grid; gap:8px; font-size:var(--text-xs); } .settings input { width:110px; }
select,input[type='number'] { box-sizing:border-box; padding:9px 12px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); font:var(--text-sm) var(--font-body); min-width:0; }
input:disabled,select:disabled { opacity:0.6; } .run-picker { max-width:100%; } .run-picker select { max-width:100%; }
.runs { display:grid; grid-template-columns:minmax(0,1fr); gap:16px; border-top:1px solid var(--border-subtle); padding-top:20px; min-width:0; }
.error { display:flex; gap:12px; align-items:center; justify-content:space-between; padding:12px 16px; color:var(--state-danger); border:1px solid var(--state-danger); border-radius:var(--radius-sm); font-size:var(--text-sm); overflow-wrap:anywhere; } .notice { color:var(--state-online); }
.text-button { color:var(--accent-primary-hover); } .text-button { border:0; padding:0; background:none; font:inherit; cursor:pointer; text-decoration:underline; }
button:focus-visible,input:focus-visible,select:focus-visible { outline:2px solid var(--accent-primary); outline-offset:3px; }
.sr-only { position:absolute; width:1px; height:1px; overflow:hidden; clip-path:inset(50%); white-space:nowrap; }
@media(max-width:600px) { .panel { padding:16px; } }
</style>
@@ -0,0 +1,41 @@
<script lang="ts">
import type { RebalancePreview } from '$lib/rebalancing.js';
import { contextHref } from '$lib/navigation-context.js';
import { timeFormat } from '$ui/stores/time-zone.js';
import { plural } from '$ui/plural.js';
import Button from '$ui/components/primitives/Button.svelte';
let { preview, expired, disabled = false, starting = false, onstart }: {
preview: RebalancePreview; expired: boolean; disabled?: boolean; starting?: boolean; onstart: () => void;
} = $props();
const numbers = new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 });
function percent(value: number) { return `${numbers.format(value)}%`; }
</script>
<section class="preview" aria-labelledby="rebalance-preview-title">
<div class="heading"><h3 id="rebalance-preview-title">Preview · {preview.moves.length} proposed {plural(preview.moves.length, 'move')}</h3><span class="help">Created {$timeFormat.dateTime(preview.createdAt)}</span></div>
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Scrollable tables must be reachable by keyboard.) -->
<div class="table-wrap" role="region" aria-label="Projected host distribution" tabindex="0"><table><thead><tr><th>Host</th><th>Instances</th><th>CPU</th><th>Memory</th></tr></thead><tbody>
{#each preview.hosts as host (host.id)}<tr><td>{host.name}</td><td>{host.before.instances} → {host.after.instances}</td><td>{percent(host.before.cpuPercent)} → {percent(host.after.cpuPercent)}</td><td>{percent(host.before.memoryPercent)} → {percent(host.after.memoryPercent)}</td></tr>{/each}
</tbody></table></div>
<p class="help">Current → projected distribution. CPU and memory estimates include the full load on each participating host.</p>
{#if preview.moves.length}<ol class="moves">{#each preview.moves as move}<li><a href={contextHref(`/servers/${encodeURIComponent(move.server)}`, '/capacity?rebalance=1')}>{move.server}</a><span>{move.sourceHostName} → {move.targetHostName}</span></li>{/each}</ol>
{:else}<p>No eligible moves would improve the distribution enough. Occupied lobbies and other excluded instances stay on their current hosts.</p>{/if}
{#if preview.warnings.length}<ul class="warnings">{#each preview.warnings as warning}<li>{warning}</li>{/each}</ul>{/if}
{#if preview.exclusions.length}<details><summary>{preview.exclusions.length} excluded {plural(preview.exclusions.length, 'instance')}</summary><ul class="exclusions">{#each preview.exclusions as exclusion}<li><strong>{exclusion.server}</strong><span>{exclusion.reason}</span></li>{/each}</ul></details>{/if}
<div class="start"><Button variant="primary" disabled={!preview.moves.length || disabled || expired} loading={starting} onclick={onstart}>Start rebalance</Button><p class="help">{expired ? 'This preview has expired. Preview again before starting.' : 'The controller rechecks occupancy and capacity before each move. Moves that no longer help are skipped.'}</p></div>
</section>
<style>
.preview { display:grid; gap:16px; border-top:1px solid var(--border-subtle); padding-top:20px; min-width:0; }
h3,p { margin:0; } h3 { font-size:var(--text-md); } p { font-size:var(--text-sm); line-height:1.6; }
.heading,.start { display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:12px; }
.help { font-size:var(--text-xs); color:var(--text-secondary); line-height:1.6; }
.table-wrap { overflow-x:auto; border:1px solid var(--border-subtle); border-radius:var(--radius-sm); }
table { width:100%; min-width:520px; border-collapse:collapse; text-align:left; font-size:var(--text-sm); font-variant-numeric:tabular-nums; }
th,td { padding:12px 16px; } th { background:var(--surface-sunken); color:var(--text-secondary); font-size:var(--text-xs); font-weight:500; } td { border-top:1px solid var(--border-subtle); }
.moves,.exclusions { margin:0; padding-left:20px; display:grid; gap:12px; font-size:var(--text-sm); } .moves span,.exclusions span { display:block; margin-top:4px; color:var(--text-secondary); font-size:var(--text-xs); } .exclusions { margin-top:12px; } .exclusions strong { font-weight:500; }
.warnings { border-left:2px solid var(--state-warning); padding:0 0 0 22px; margin:0; color:var(--text-secondary); font-size:var(--text-xs); line-height:1.7; }
summary { cursor:pointer; color:var(--text-secondary); font-size:var(--text-sm); }
.start { justify-content:flex-start; } .start p { flex:1 1 280px; }
a { color:var(--accent-primary-hover); } a:focus-visible,summary:focus-visible,.table-wrap:focus-visible { outline:2px solid var(--accent-primary); outline-offset:3px; }
</style>
@@ -0,0 +1,27 @@
<script lang="ts">
import type { RebalanceRun } from '$lib/rebalancing.js';
import { contextHref } from '$lib/navigation-context.js';
import { rebalanceMoveLabels, rebalanceRunLabels, rebalanceRunHref } from '$ui/rebalancing-presentation.js';
import Button from '$ui/components/primitives/Button.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
let { run, canManage, busy = false, stopping = false, onstop }: {
run: RebalanceRun; canManage: boolean; busy?: boolean; stopping?: boolean; onstop: () => void;
} = $props();
</script>
<div class="heading"><div class="run-status"><Badge variant={run.state === 'failed' ? 'danger' : run.state === 'done' ? 'online' : run.state === 'stopped' ? 'neutral' : 'info'}>{rebalanceRunLabels[run.state]}</Badge><span>{run.moves.filter(move => move.state === 'done').length} of {run.moves.length} moves completed</span><a href={rebalanceRunHref(run.id)}>Link to run</a></div>
{#if canManage && run.state === 'running'}<Button disabled={busy} loading={stopping} onclick={onstop}>Stop remaining moves</Button>{/if}</div>
<p role="status">{run.message}</p>
{#if run.state === 'running' || run.state === 'stopping'}<p class="help">Stopping withdraws unstarted moves. A move already in progress continues through health verification.</p>{/if}
<ol class="run-moves">{#each run.moves as move (move.id)}<li><div class="heading"><div class="move-identity"><strong>{move.server}</strong><span>{move.sourceHostName} → {move.targetHostName}</span></div><Badge variant={move.state === 'failed' ? 'danger' : move.state === 'done' ? 'online' : move.state === 'moving' ? 'info' : 'neutral'}>{rebalanceMoveLabels[move.state]}</Badge></div><p>{move.message}</p>{#if move.operationId}<a href={contextHref(`/provisioning/${encodeURIComponent(move.operationId)}`, rebalanceRunHref(run.id))}>View provisioning →</a>{/if}</li>{/each}</ol>
<style>
p { margin:0; font-size:var(--text-sm); line-height:1.6; }
.heading,.run-status { display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:12px; }
.help { font-size:var(--text-xs); color:var(--text-secondary); line-height:1.6; }
.run-status { justify-content:flex-start; font-size:var(--text-xs); color:var(--text-secondary); } .run-moves { margin:0; padding:0; list-style:none; display:grid; gap:12px; }
.run-moves li { display:grid; gap:8px; padding:16px; border:1px solid var(--border-subtle); border-radius:var(--radius-sm); } .run-moves p,.run-moves a { font-size:var(--text-xs); } .run-moves p { color:var(--text-secondary); }
.move-identity { display:grid; gap:5px; font-size:var(--text-sm); min-width:0; overflow-wrap:anywhere; } .move-identity strong { font-weight:500; } .move-identity span { font-size:var(--text-xs); color:var(--text-secondary); }
a { color:var(--accent-primary-hover); } a:focus-visible { outline:2px solid var(--accent-primary); outline-offset:3px; }
@media(max-width:600px) { .run-moves li { padding:12px; } }
</style>
+7 -1
View File
@@ -119,6 +119,12 @@ export function provisioningReturn(value: string | null | undefined): { href: st
}
return { href: '/provisioning' + (params.size ? `?${params}` : ''), label: 'Back to Provisioning' };
}
if (url.pathname === '/capacity' && url.searchParams.get('rebalance') === '1') {
const params = new URLSearchParams({ rebalance: '1' });
const run = url.searchParams.get('rebalanceRun');
if (run) params.set('rebalanceRun', run);
return { href: `/capacity?${params}`, label: 'Back to rebalance' };
}
const match = url.pathname.match(/^\/(campaigns|betas|maintenance|collections\/launches)\/[^/]+$/);
if (match) return { href: url.pathname, label: `Back to ${match[1] === 'collections/launches' ? 'collection' : match[1] === 'campaigns' ? 'campaign' : match[1] === 'betas' ? 'beta' : 'maintenance'}` };
} catch { /* Fall back to the local list. */ }
@@ -180,7 +186,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
if (!record(value.activity) || !number(value.activity.total) || !(value.activity.latestId === null || number(value.activity.latestId))
|| !(value.activity.failureExcerpt === null || event(value.activity.failureExcerpt))) return false;
if (!Array.isArray(value.sources) || !value.sources.every(source => record(source)
&& ['campaign', 'beta', 'maintenance', 'watchdog', 'autoscaling'].includes(source.kind as string)
&& ['campaign', 'beta', 'maintenance', 'watchdog', 'autoscaling', 'rebalance'].includes(source.kind as string)
&& text(source.source_id) && text(source.label) && localLink(source.href))) return false;
if (!(value.collection === null || record(value.collection) && text(value.collection.id) && text(value.collection.name))) return false;
if (!record(value.capacity) || !timestamp(value.capacity.evaluatedAt) || !Array.isArray(value.capacity.pools)
+9
View File
@@ -0,0 +1,9 @@
import type { RebalanceMove, RebalanceRun } from '$lib/rebalancing.js';
export const rebalanceRunLabels: Record<RebalanceRun['state'], string> = {
running: 'In progress', stopping: 'Stopping', done: 'Finished', stopped: 'Stopped', failed: 'Needs attention'
};
export const rebalanceMoveLabels: Record<RebalanceMove['state'], string> = {
planned: 'Not started', moving: 'Moving', done: 'Completed', skipped: 'Skipped', failed: 'Needs attention'
};
export function rebalanceRunHref(id: string) { return `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}`; }