Add safe manual instance rebalancing

This commit is contained in:
2026-10-08 12:37:56 -04:00
parent 26de43639f
commit 012cd51a66
45 changed files with 2006 additions and 154 deletions
+92
View File
@@ -0,0 +1,92 @@
import { describe, expect, it } from 'vitest';
import { rebalanceSettings, type RebalanceSettings } from '../../src/lib/rebalancing.js';
import { planRebalance, rebalanceMoveBlocker, type RebalancingHost, type RebalancingInstance, type RebalancingSnapshot } from '../../src/lib/server/rebalancing/planner.js';
const gib = 1024 ** 3;
const settings: RebalanceSettings = { hostIds: ['a', 'b'], game: 'all', maxMoves: 20 };
const host = (id: string, cpu: number, count = 0): RebalancingHost => ({ id, name: id, cpuCount: 8, memoryTotal: 16 * gib,
used: { cpuCores: cpu, memoryBytes: gib }, running: { cpuCores: 0.5, memoryBytes: gib },
startup: { cpuCores: 1, memoryBytes: 2 * gib }, instances: count, ports: 20, reason: null });
const instance = (n: number, source = 'a', cpu = 1): RebalancingInstance => ({ server: `t6_zombie_town_${n}`, instanceId: `i${n}`, generation: 1,
sourceHostId: source, game: 't6', relief: { cpuCores: cpu, memoryBytes: gib }, demand: { cpuCores: cpu, memoryBytes: gib }, destinations: { a: null, b: null } });
const snapshot = (hosts = [host('a', 6, 6), host('b', 0)], instances = Array.from({ length: 6 }, (_, n) => instance(n))): RebalancingSnapshot => ({ hosts, instances, exclusions: [], warnings: [] });
describe('rebalance planning', () => {
it('validates bounded hosts and moves without silently choosing all hosts', () => {
expect(rebalanceSettings({ ...settings, hostIds: ['b', 'a'] }).hostIds).toEqual(['a', 'b']);
for (const input of [{}, { ...settings, hostIds: ['a'] }, { ...settings, hostIds: ['a', 'a'] }, { ...settings, game: 't4' },
{ ...settings, maxMoves: 0 }, { ...settings, maxMoves: 21 }, { ...settings, maxMoves: 2.5 }]) expect(() => rebalanceSettings(input)).toThrow();
});
it('spreads pressure with the fewest useful moves and never mutates live measurements', () => {
const input = snapshot(), saved = structuredClone(input);
const result = planRebalance(input, settings);
expect(result.moves.map(move => [move.server, move.sourceHostId, move.targetHostId])).toEqual([0, 1, 2].map(n => [`t6_zombie_town_${n}`, 'a', 'b']));
expect(result.hosts.map(item => item.after.instances)).toEqual([3, 3]);
expect(result.hosts.map(item => item.after.cpuPercent)).toEqual([37.5, 37.5]);
expect(new Set(result.moves.map(move => move.instanceId)).size).toBe(result.moves.length);
expect(input).toEqual(saved);
});
it('uses host size and both resource dimensions rather than equal counts', () => {
const small = host('a', 3, 3); small.cpuCount = 4;
const large = host('b', 4, 4); large.cpuCount = 16;
expect(planRebalance(snapshot([small, large], [instance(1)]), settings).moves).toHaveLength(1);
small.used = { cpuCores: 1, memoryBytes: 12 * gib }; large.used = { cpuCores: 1, memoryBytes: 2 * gib };
const memory = instance(1); memory.relief.memoryBytes = 2 * gib; memory.demand.memoryBytes = 2 * gib;
expect(planRebalance(snapshot([small, large], [memory]), settings).moves[0].improvement).toBeCloseTo(12.5);
});
it('respects startup fit, port capacity, compatibility, host eligibility, and reservations', () => {
for (const mutate of [
(target: RebalancingHost) => { target.ports = 0; },
(target: RebalancingHost) => { target.reason = 'Host disabled.'; },
(target: RebalancingHost) => { target.startup.cpuCores = 8; },
(target: RebalancingHost) => { target.startup.memoryBytes = 16 * gib; },
(target: RebalancingHost) => { target.used.cpuCores = 7; }
]) {
const input = snapshot(); mutate(input.hosts[1]);
expect(planRebalance(input, settings).moves).toEqual([]);
}
const input = snapshot(); input.instances.forEach(item => { item.destinations.b = 'Missing the deployed runtime.'; });
expect(planRebalance(input, settings).moves).toEqual([]);
expect(planRebalance(input, settings).exclusions[0].reason).toContain('Missing the deployed runtime');
input.instances.forEach(item => { delete item.destinations.b; });
expect(planRebalance(input, settings).moves).toEqual([]);
});
it('requires five percentage points of improvement, preserves limits, and does not move twice', () => {
const input = snapshot([host('a', 4), host('b', 2)], [instance(1, 'a', 0.3)]);
expect(planRebalance(input, settings).moves).toEqual([]);
input.instances[0].relief.cpuCores = 0.4; input.instances[0].demand.cpuCores = 0.4;
expect(planRebalance(input, settings).moves[0].improvement).toBeCloseTo(5);
expect(planRebalance(snapshot(), { ...settings, maxMoves: 1 }).moves).toHaveLength(1);
expect(planRebalance(snapshot(), { ...settings, game: 't5' }).moves).toHaveLength(0);
const onePort = snapshot(); onePort.hosts[1].ports = 1;
expect(planRebalance(onePort, settings).moves).toHaveLength(1);
});
it('prefers the largest improvement deterministically and revalidates the approved pair', () => {
const input = snapshot([host('a', 6, 6), host('b', 0)], [instance(3), instance(2, 'a', 2), instance(1)]);
const move = planRebalance(input, { ...settings, maxMoves: 1 }).moves[0];
expect(move.server).toBe('t6_zombie_town_2');
expect(rebalanceMoveBlocker(input, move)).toBeNull();
expect(rebalanceMoveBlocker(input, { ...move, generation: 2 })).toContain('generation has changed');
input.hosts[1].used.cpuCores = 6;
expect(rebalanceMoveBlocker(input, move)).toBeTruthy();
input.instances = []; input.exclusions.push({ server: move.server, reason: 'Players are present.' });
expect(rebalanceMoveBlocker(input, move)).toBe('Players are present.');
});
it('only credits source ports actually released inside the configured range and bind', () => {
const a = host('a', 6.8, 1), b = host('b', 5.2, 1), c = host('c', 0);
a.ports = 0; b.ports = 0; c.ports = 1;
const first = instance(1, 'a', 4), second = instance(2, 'b', 1);
first.destinations.c = null; second.destinations.c = null;
const input = snapshot([a, b, c], [first, second]);
const selection = { ...settings, hostIds: ['a', 'b', 'c'] };
expect(planRebalance(input, selection).moves).toHaveLength(1);
first.releasesPort = true;
expect(planRebalance(input, selection).moves).toHaveLength(2);
});
});