Add safe manual instance rebalancing
This commit is contained in:
@@ -7,7 +7,7 @@ const allows = (role: UserRole, route: string | null, method = 'GET') => roleAll
|
||||
|
||||
describe('management route permissions', () => {
|
||||
it.each([
|
||||
'/', '/api/fleet', '/capacity', '/api/capacity', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/', '/api/fleet', '/capacity', '/api/capacity', '/api/rebalancing', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts',
|
||||
'/api/metrics', '/metrics', '/api/matches', '/matches/[id]',
|
||||
'/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]',
|
||||
'/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events',
|
||||
@@ -22,6 +22,7 @@ describe('management route permissions', () => {
|
||||
|
||||
it.each([
|
||||
['/api/servers', 'POST'], ['/api/servers/[name]', 'DELETE'], ['/api/provisioning', 'POST'],
|
||||
['/api/rebalancing', 'POST'],
|
||||
['/api/provisioning/[id]/cancel', 'POST'], ['/api/provisioning/[id]/resume', 'POST'],
|
||||
['/api/deployment-collections', 'POST'], ['/api/deployment-collections/preview', 'POST'],
|
||||
['/api/deployment-collections/[id]/cancel', 'POST'],
|
||||
|
||||
@@ -11,6 +11,9 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
|
||||
rebalanceAdmission: vi.fn(() => null), requestRebalanceStopForOperation: vi.fn()
|
||||
}));
|
||||
|
||||
let queuedKeys = 0;
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
@@ -84,6 +87,123 @@ async function observations(server: string) {
|
||||
return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
|
||||
}
|
||||
|
||||
async function moveTarget() {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
const target = hosts.createHost({ name: 'Rebalance destination', portMin: 29000, portMax: 29005 });
|
||||
hosts.enrollHost(hosts.createEnrollment(target.id).token, inventory());
|
||||
libraries.updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready',
|
||||
manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]);
|
||||
return target;
|
||||
}
|
||||
|
||||
describe('rebalance instance migrations', () => {
|
||||
it('preserves deployed configuration, patched file versions, identity and credentials on an active source', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
Object.assign(config.rcon, { password: '' });
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const baseline = createScript({ name: 'Deployed baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5',
|
||||
content: Buffer.from('set sv_voice "1"\n') });
|
||||
const script = createScript({ name: 'Deployed script', kind: 'script', targetRelPath: 'preserved.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('original deployed script') });
|
||||
for (const artifact of [baseline, script]) db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id);
|
||||
const job = await queue(); await advance();
|
||||
const launch = (await operationFor(job.deploymentId))!.payload as { rconPassword: string; files: { path: string; sha256: string }[] };
|
||||
await acknowledge(job.deploymentId);
|
||||
const patched = uploadNewVersion(script.script.id, Buffer.from('confirmed patch after initial launch'));
|
||||
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
recordScript(job.name, 'preserved.gsc', patched.id, patched.sha256, patched.size, 'campaign');
|
||||
const newest = uploadNewVersion(script.script.id, Buffer.from('new promotion must not be installed'));
|
||||
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newest.id, script.script.id);
|
||||
const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n'));
|
||||
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newBaseline.id, baseline.script.id);
|
||||
Object.assign(config.rcon, { password: 'new-controller-password' });
|
||||
const target = await moveTarget();
|
||||
const { captureMoveRecipe, requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const original = nativeInstance(job.name)!;
|
||||
const recipe = captureMoveRecipe(job.name);
|
||||
expect(recipe.files.every(file => file.content === undefined)).toBe(true);
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test', recipe });
|
||||
await advance();
|
||||
const removal = hostMigration(id)!.remove_operation_id!;
|
||||
expect(removal).toBeTruthy();
|
||||
const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(canCancelProvisioningJob(id)).toBe(false);
|
||||
expect(cancelProvisioningJob(id)).toBe(false);
|
||||
const { getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)?.canCancel).toBe(false);
|
||||
expect(listDeployments().find(item => item.id === id)?.canCancel).toBe(0);
|
||||
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
|
||||
const response = POST({ params: { id } });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ error: expect.stringContaining('must finish recreating') });
|
||||
const { reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, removal, { state: 'completed' });
|
||||
await advance(); await advance();
|
||||
const creation = (await operationFor(id))!;
|
||||
expect(creation).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2,
|
||||
payload: { name: job.name, libraryId, image, rconPassword: launch.rconPassword, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
|
||||
const payload = creation.payload as { files: { path: string; sha256: string }[] };
|
||||
expect(payload.files).toContainEqual(expect.objectContaining({ path: expect.stringContaining('preserved.gsc'), sha256: patched.sha256 }));
|
||||
expect(payload.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256)
|
||||
.toBe(launch.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256);
|
||||
reportHostOperation(target.id, creation.id, { state: 'completed', result: { state: 'running', healthy: true } });
|
||||
await advance();
|
||||
expect((await detail(id)).state).toBe('done');
|
||||
expect(nativeInstance(job.name)).toMatchObject({ id: original.id, host_id: target.id, generation: 2 });
|
||||
expect(getHost(hostId)?.mode).toBe('active');
|
||||
expect(JSON.stringify(getDeployment(id)?.recipe)).not.toContain(launch.rconPassword);
|
||||
});
|
||||
|
||||
it('rechecks occupancy, target capacity and run admission before removing the source', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const target = await moveTarget();
|
||||
const { requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
expect(() => requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' })).toThrow('different destination');
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
||||
remote.players = 2; await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
remote.players = 0;
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(target.id, { ...inventory(), memoryAvailable: 1 });
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
heartbeat(target.id, inventory());
|
||||
const { rebalanceAdmission } = await import('../../src/lib/server/rebalancing/admission.js');
|
||||
vi.mocked(rebalanceAdmission).mockReturnValueOnce('Run stopped during occupancy check.');
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('withdraws only uncommitted moves and preserves their server keys', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const target = await moveTarget();
|
||||
const { requestInstanceMove } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
const { requestRebalanceStopForOperation } = await import('../../src/lib/server/rebalancing/admission.js');
|
||||
expect(requestRebalanceStopForOperation).toHaveBeenCalledWith(id);
|
||||
await advance();
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(job.name)).toBe('saved-key');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses to move when recorded deployed bytes cannot be restored', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { captureMoveRecipe } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE server_scripts SET sha256=? WHERE server=?").run('0'.repeat(64), job.name);
|
||||
expect(() => captureMoveRecipe(job.name)).toThrow('deployed bytes are unavailable');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runtime configuration at dispatch', () => {
|
||||
it.each([
|
||||
['t5', 'shared'], ['t5', 'generated'], ['t6', 'shared'], ['t6', 'generated']
|
||||
|
||||
@@ -163,6 +163,18 @@ describe('provisioning response and return-context boundaries', () => {
|
||||
expect(provisioningReturn('/collections/launches/launch-a').href).toBe('/collections/launches/launch-a');
|
||||
});
|
||||
|
||||
it('returns rebalance moves to their run while discarding unrelated navigation parameters', () => {
|
||||
expect(provisioningReturn('/capacity?rebalance=1&rebalanceRun=run-a&redirect=https://evil.test'))
|
||||
.toEqual({ href: '/capacity?rebalance=1&rebalanceRun=run-a', label: 'Back to rebalance' });
|
||||
expect(provisioningReturn('/capacity?rebalance=1')).toEqual({ href: '/capacity?rebalance=1', label: 'Back to rebalance' });
|
||||
expect(provisioningReturn('/capacity?rebalanceRun=run-a').href).toBe('/provisioning');
|
||||
const detail = fixture();
|
||||
detail.sources = [{ kind: 'rebalance', source_id: 'run-a', label: 'Rebalance', href: '/capacity?rebalance=1&rebalanceRun=run-a' }];
|
||||
expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
|
||||
detail.sources[0].href = '//evil.test';
|
||||
expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects incomplete nested snapshots before they replace a working detail', () => {
|
||||
const detail = fixture();
|
||||
expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { rebalanceSettings, type RebalanceSettings } from '../../src/lib/rebalancing.js';
|
||||
import { planRebalance, rebalanceMoveBlocker, type RebalancingHost, type RebalancingInstance, type RebalancingSnapshot } from '../../src/lib/server/rebalancing/planner.js';
|
||||
|
||||
const gib = 1024 ** 3;
|
||||
const settings: RebalanceSettings = { hostIds: ['a', 'b'], game: 'all', maxMoves: 20 };
|
||||
const host = (id: string, cpu: number, count = 0): RebalancingHost => ({ id, name: id, cpuCount: 8, memoryTotal: 16 * gib,
|
||||
used: { cpuCores: cpu, memoryBytes: gib }, running: { cpuCores: 0.5, memoryBytes: gib },
|
||||
startup: { cpuCores: 1, memoryBytes: 2 * gib }, instances: count, ports: 20, reason: null });
|
||||
const instance = (n: number, source = 'a', cpu = 1): RebalancingInstance => ({ server: `t6_zombie_town_${n}`, instanceId: `i${n}`, generation: 1,
|
||||
sourceHostId: source, game: 't6', relief: { cpuCores: cpu, memoryBytes: gib }, demand: { cpuCores: cpu, memoryBytes: gib }, destinations: { a: null, b: null } });
|
||||
const snapshot = (hosts = [host('a', 6, 6), host('b', 0)], instances = Array.from({ length: 6 }, (_, n) => instance(n))): RebalancingSnapshot => ({ hosts, instances, exclusions: [], warnings: [] });
|
||||
|
||||
describe('rebalance planning', () => {
|
||||
it('validates bounded hosts and moves without silently choosing all hosts', () => {
|
||||
expect(rebalanceSettings({ ...settings, hostIds: ['b', 'a'] }).hostIds).toEqual(['a', 'b']);
|
||||
for (const input of [{}, { ...settings, hostIds: ['a'] }, { ...settings, hostIds: ['a', 'a'] }, { ...settings, game: 't4' },
|
||||
{ ...settings, maxMoves: 0 }, { ...settings, maxMoves: 21 }, { ...settings, maxMoves: 2.5 }]) expect(() => rebalanceSettings(input)).toThrow();
|
||||
});
|
||||
|
||||
it('spreads pressure with the fewest useful moves and never mutates live measurements', () => {
|
||||
const input = snapshot(), saved = structuredClone(input);
|
||||
const result = planRebalance(input, settings);
|
||||
expect(result.moves.map(move => [move.server, move.sourceHostId, move.targetHostId])).toEqual([0, 1, 2].map(n => [`t6_zombie_town_${n}`, 'a', 'b']));
|
||||
expect(result.hosts.map(item => item.after.instances)).toEqual([3, 3]);
|
||||
expect(result.hosts.map(item => item.after.cpuPercent)).toEqual([37.5, 37.5]);
|
||||
expect(new Set(result.moves.map(move => move.instanceId)).size).toBe(result.moves.length);
|
||||
expect(input).toEqual(saved);
|
||||
});
|
||||
|
||||
it('uses host size and both resource dimensions rather than equal counts', () => {
|
||||
const small = host('a', 3, 3); small.cpuCount = 4;
|
||||
const large = host('b', 4, 4); large.cpuCount = 16;
|
||||
expect(planRebalance(snapshot([small, large], [instance(1)]), settings).moves).toHaveLength(1);
|
||||
small.used = { cpuCores: 1, memoryBytes: 12 * gib }; large.used = { cpuCores: 1, memoryBytes: 2 * gib };
|
||||
const memory = instance(1); memory.relief.memoryBytes = 2 * gib; memory.demand.memoryBytes = 2 * gib;
|
||||
expect(planRebalance(snapshot([small, large], [memory]), settings).moves[0].improvement).toBeCloseTo(12.5);
|
||||
});
|
||||
|
||||
it('respects startup fit, port capacity, compatibility, host eligibility, and reservations', () => {
|
||||
for (const mutate of [
|
||||
(target: RebalancingHost) => { target.ports = 0; },
|
||||
(target: RebalancingHost) => { target.reason = 'Host disabled.'; },
|
||||
(target: RebalancingHost) => { target.startup.cpuCores = 8; },
|
||||
(target: RebalancingHost) => { target.startup.memoryBytes = 16 * gib; },
|
||||
(target: RebalancingHost) => { target.used.cpuCores = 7; }
|
||||
]) {
|
||||
const input = snapshot(); mutate(input.hosts[1]);
|
||||
expect(planRebalance(input, settings).moves).toEqual([]);
|
||||
}
|
||||
const input = snapshot(); input.instances.forEach(item => { item.destinations.b = 'Missing the deployed runtime.'; });
|
||||
expect(planRebalance(input, settings).moves).toEqual([]);
|
||||
expect(planRebalance(input, settings).exclusions[0].reason).toContain('Missing the deployed runtime');
|
||||
input.instances.forEach(item => { delete item.destinations.b; });
|
||||
expect(planRebalance(input, settings).moves).toEqual([]);
|
||||
});
|
||||
|
||||
it('requires five percentage points of improvement, preserves limits, and does not move twice', () => {
|
||||
const input = snapshot([host('a', 4), host('b', 2)], [instance(1, 'a', 0.3)]);
|
||||
expect(planRebalance(input, settings).moves).toEqual([]);
|
||||
input.instances[0].relief.cpuCores = 0.4; input.instances[0].demand.cpuCores = 0.4;
|
||||
expect(planRebalance(input, settings).moves[0].improvement).toBeCloseTo(5);
|
||||
expect(planRebalance(snapshot(), { ...settings, maxMoves: 1 }).moves).toHaveLength(1);
|
||||
expect(planRebalance(snapshot(), { ...settings, game: 't5' }).moves).toHaveLength(0);
|
||||
const onePort = snapshot(); onePort.hosts[1].ports = 1;
|
||||
expect(planRebalance(onePort, settings).moves).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('prefers the largest improvement deterministically and revalidates the approved pair', () => {
|
||||
const input = snapshot([host('a', 6, 6), host('b', 0)], [instance(3), instance(2, 'a', 2), instance(1)]);
|
||||
const move = planRebalance(input, { ...settings, maxMoves: 1 }).moves[0];
|
||||
expect(move.server).toBe('t6_zombie_town_2');
|
||||
expect(rebalanceMoveBlocker(input, move)).toBeNull();
|
||||
expect(rebalanceMoveBlocker(input, { ...move, generation: 2 })).toContain('generation has changed');
|
||||
input.hosts[1].used.cpuCores = 6;
|
||||
expect(rebalanceMoveBlocker(input, move)).toBeTruthy();
|
||||
input.instances = []; input.exclusions.push({ server: move.server, reason: 'Players are present.' });
|
||||
expect(rebalanceMoveBlocker(input, move)).toBe('Players are present.');
|
||||
});
|
||||
|
||||
it('only credits source ports actually released inside the configured range and bind', () => {
|
||||
const a = host('a', 6.8, 1), b = host('b', 5.2, 1), c = host('c', 0);
|
||||
a.ports = 0; b.ports = 0; c.ports = 1;
|
||||
const first = instance(1, 'a', 4), second = instance(2, 'b', 1);
|
||||
first.destinations.c = null; second.destinations.c = null;
|
||||
const input = snapshot([a, b, c], [first, second]);
|
||||
const selection = { ...settings, hostIds: ['a', 'b', 'c'] };
|
||||
expect(planRebalance(input, selection).moves).toHaveLength(1);
|
||||
first.releasesPort = true;
|
||||
expect(planRebalance(input, selection).moves).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import type { RebalanceMove } from '../../src/lib/rebalancing.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } })),
|
||||
rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
||||
}));
|
||||
|
||||
let fixture: Fixture, sourceId: string, targetId: string, libraryId: string;
|
||||
const names = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`);
|
||||
const image = 'example.test/plutainer:rebalance';
|
||||
const gib = 1024 ** 3;
|
||||
const clockStep = () => vi.setSystemTime(Date.now() + 1000);
|
||||
const settings = () => ({ hostIds: [sourceId, targetId], game: 'all' as const, maxMoves: 5 });
|
||||
|
||||
async function heartbeat(hostId: string, cpuUsagePercent: number) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const instances = db().prepare('SELECT id,generation FROM native_instances WHERE host_id=? AND retired_at IS NULL')
|
||||
.all(hostId) as { id: string; generation: number }[];
|
||||
heartbeat(hostId, { protocolVersion: 1, cpuCount: 16, cpuUsagePercent, memoryTotal: 32 * gib,
|
||||
memoryAvailable: 32 * gib - (2 + instances.length * 2) * gib, dockerRunning: true,
|
||||
directoriesWritable: true, storageAvailable: 100 * gib, runtimeImages: [image],
|
||||
libraries: [{ id: libraryId, state: 'ready' }], instanceResources: instances.map(instance => ({
|
||||
instanceId: instance.id, generation: instance.generation, uptimeSeconds: 3600, cpuCores: 3, memoryBytes: 2 * gib })) });
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-10-08T16:00:00.000Z'));
|
||||
fixture = await newFixture([]); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'preserved-password');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const at = new Date().toISOString(), launched = new Date(Date.now() - 600_000).toISOString();
|
||||
for (const [index, name] of names.entries()) db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,
|
||||
config_file,game_id,image,game_port,container_state,container_checked_at,last_players,last_map,last_seen_at,last_checked_at,last_roster)
|
||||
VALUES(?,'t5','zombie_theater','zombie','theater','dedicated_sp.cfg','t5sp',?,?,'running',?,0,'zombie_theater',?,?,'[]')`)
|
||||
.run(name, image, 28960 + index, at, at, at);
|
||||
const prepared = await prepareNativeHost(names); sourceId = prepared.host.id; libraryId = prepared.versions.get('t5')!;
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
targetId = hosts.createHost({ name: 'Empty destination', portMin: 29000, portMax: 29020 }).id;
|
||||
hosts.enrollHost(hosts.createEnrollment(targetId).token, { protocolVersion: 1, cpuCount: 16, cpuUsagePercent: 6.25,
|
||||
memoryTotal: 32 * gib, memoryAvailable: 30 * gib, dockerRunning: true, directoriesWritable: true,
|
||||
storageAvailable: 100 * gib, runtimeImages: [image], libraries: [{ id: libraryId, state: 'ready' }] });
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
libraries.updateHostLibraryCaches(targetId, [{ libraryVersionId: libraryId, status: 'ready',
|
||||
manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]);
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { captureRecipe, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
for (const name of names) {
|
||||
const instance = nativeInstance(name)!;
|
||||
reserveServerKey(name, 't5', `saved-key-${name}`);
|
||||
for (const protocol of ['udp', 'tcp'] as const) hosts.reserveHostPort(sourceId, instance.id, { port: instance.port, bindAddress: '0.0.0.0', protocol });
|
||||
const id = beginDeployment(name, 'create', 'fixture');
|
||||
const operation = hosts.queueHostOperation(sourceId, 'instance.create', { name, rconPassword: 'preserved-password' },
|
||||
{ instanceId: instance.id, generation: 1, idempotencyKey: randomUUID() });
|
||||
hosts.reportHostOperation(sourceId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } });
|
||||
db().prepare("UPDATE server_deployments SET state='done',submitted_at=?,finished_at=? WHERE id=?").run(launched, launched, id);
|
||||
db().prepare('UPDATE native_provisioning SET operation_id=?,verified_at=? WHERE deployment_id=?').run(operation.id, launched, id);
|
||||
const recipe = captureRecipe('t5', 'theater', 'preserved-password');
|
||||
const stored = recipeStorage({ ...recipe, image, native: { hostId: sourceId, libraryId, image } });
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
|
||||
for (const file of recipe.files) recordScript(name, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
||||
}
|
||||
await heartbeat(sourceId, 75); await heartbeat(targetId, 6.25);
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
||||
if (kind !== 'instance.inspect') throw new Error(`Unexpected host request ${kind}`);
|
||||
return { state: 'running', healthy: true, exists: true } as never;
|
||||
});
|
||||
});
|
||||
afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function start() {
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
const preview = service.previewRebalance(settings());
|
||||
expect(preview.moves.length, JSON.stringify(preview.exclusions)).toBeGreaterThanOrEqual(2);
|
||||
return service.startRebalance(preview.id);
|
||||
}
|
||||
async function currentMove(runId: string): Promise<RebalanceMove> {
|
||||
const { getRebalanceRun } = await import('../../src/lib/server/rebalancing/index.js');
|
||||
return getRebalanceRun(runId)!.moves.find(move => move.state === 'moving')!;
|
||||
}
|
||||
async function provision() {
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
}
|
||||
async function finishMove(move: RebalanceMove) {
|
||||
const { hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const removal = hostMigration(move.operationId!)!.remove_operation_id;
|
||||
expect(removal).toBeTruthy();
|
||||
clockStep(); hosts.reportHostOperation(sourceId, removal!, { state: 'completed' });
|
||||
await provision(); await provision();
|
||||
const created = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(move.operationId) as { operation_id: string | null };
|
||||
expect(created.operation_id).toBeTruthy();
|
||||
clockStep(); hosts.reportHostOperation(targetId, created.operation_id!, { state: 'completed', result: { state: 'running', healthy: true } });
|
||||
clockStep(); await provision();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId)).toEqual({ state: 'done' });
|
||||
}
|
||||
|
||||
describe('manual rebalance runs', () => {
|
||||
it('previews without reserving capacity and rejects stale plans', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { previewRebalance, startRebalance } = await import('../../src/lib/server/rebalancing/index.js');
|
||||
const ports = db().prepare('SELECT * FROM ec_host_ports').all();
|
||||
const commands = db().prepare('SELECT id FROM ec_host_operations').all();
|
||||
const preview = previewRebalance(settings());
|
||||
expect(preview.moves.length).toBeGreaterThanOrEqual(2);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual(ports);
|
||||
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual(commands);
|
||||
expect(db().prepare('SELECT * FROM rebalancing_runs').all()).toEqual([]);
|
||||
db().prepare("UPDATE servers SET desired_state='stopped' WHERE name=?").run(preview.moves[0].server);
|
||||
expect(() => startRebalance(preview.id)).toThrow('fleet changed');
|
||||
});
|
||||
|
||||
it('starts idempotently, rejects concurrent runs, and queues exactly one move', async () => {
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
const preview = service.previewRebalance(settings());
|
||||
const competing = service.previewRebalance(settings());
|
||||
const run = service.startRebalance(preview.id);
|
||||
expect(service.startRebalance(preview.id).id).toBe(run.id);
|
||||
expect(() => service.startRebalance(competing.id)).toThrow('current rebalance');
|
||||
expect(await service.runRebalancing()).toBe(true);
|
||||
expect(await service.runRebalancing()).toBe(false);
|
||||
const moves = service.getRebalanceRun(run.id)!.moves;
|
||||
expect(moves.filter(move => move.operationId)).toHaveLength(1);
|
||||
expect(moves.filter(move => move.state === 'planned').length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('rejects expired previews but returns an already-started run on a late retry', async () => {
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
const first = service.previewRebalance(settings());
|
||||
const expired = service.previewRebalance(settings());
|
||||
const run = service.startRebalance(first.id);
|
||||
vi.setSystemTime(Date.now() + 6 * 60_000);
|
||||
expect(service.startRebalance(first.id).id).toBe(run.id);
|
||||
expect(() => service.startRebalance(expired.id)).toThrow('Preview expired');
|
||||
});
|
||||
|
||||
it('stops before removal without changing placement or releasing the key, and allows a later host drain', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const stopped = service.stopRebalance(run.id);
|
||||
expect(stopped.state).toBe('stopped');
|
||||
expect(stopped.moves.every(move => move.state === 'skipped')).toBe(true);
|
||||
await provision();
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(nativeInstance(move.server)?.host_id).toBe(sourceId);
|
||||
expect(getServerKey(move.server)).toBe(`saved-key-${move.server}`);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
|
||||
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
updateHost(sourceId, { mode: 'draining' });
|
||||
const { reconcileHostDrains } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
reconcileHostDrains();
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE server=? AND triggered_by='host-drain'").get(move.server)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('finishes a committed move after Stop while withdrawing the remaining moves', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing(); await provision();
|
||||
const move = await currentMove(run.id);
|
||||
expect(service.stopRebalance(run.id).state).toBe('stopping');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(move.operationId!)).toBe(false);
|
||||
await finishMove(move); await service.runRebalancing();
|
||||
const finished = service.getRebalanceRun(run.id)!;
|
||||
expect(finished.state).toBe('stopped');
|
||||
expect(finished.moves.filter(move => move.state === 'done')).toHaveLength(1);
|
||||
expect(finished.moves.filter(move => move.state === 'skipped')).toHaveLength(run.moves.length - 1);
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
expect(nativeInstance(move.server)).toMatchObject({ id: move.instanceId, generation: 2, host_id: targetId });
|
||||
expect(getHost(sourceId)?.mode).toBe('active');
|
||||
});
|
||||
|
||||
it('treats cancellation from Provisioning as stopping the rest of the run', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(move.operationId!)).toBe(true);
|
||||
expect(service.getRebalanceRun(run.id)?.state).toBe('stopping');
|
||||
await service.runRebalancing();
|
||||
expect(service.getRebalanceRun(run.id)?.state).toBe('stopped');
|
||||
expect(service.getRebalanceRun(run.id)?.moves.every(move => move.state === 'skipped')).toBe(true);
|
||||
});
|
||||
|
||||
it('waits for both hosts to report fresh measurements before admitting the next move', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing(); await provision();
|
||||
await finishMove(await currentMove(run.id));
|
||||
expect(await service.runRebalancing()).toBe(false);
|
||||
expect(service.getRebalanceRun(run.id)?.message).toContain('fresh source and destination');
|
||||
expect(service.getRebalanceRun(run.id)?.moves.filter(move => move.operationId)).toHaveLength(1);
|
||||
clockStep(); await heartbeat(sourceId, 56.25);
|
||||
expect(await service.runRebalancing()).toBe(false);
|
||||
await heartbeat(targetId, 25);
|
||||
expect(await service.runRebalancing()).toBe(true);
|
||||
expect(service.getRebalanceRun(run.id)?.moves.filter(move => move.operationId)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips an uncommitted move that becomes occupied and continues other approved work', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE servers SET last_players=2 WHERE name=?').run(move.server);
|
||||
await service.runRebalancing();
|
||||
const fresh = service.getRebalanceRun(run.id)!;
|
||||
expect(fresh.state).toBe('running');
|
||||
expect(fresh.moves.find(item => item.id === move.id)?.state).toBe('skipped');
|
||||
expect(fresh.moves.filter(item => item.state === 'moving')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('halts remaining moves after a provisioning failure', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE server_deployments SET state='failed',error='Recovery needed' WHERE id=?").run(move.operationId);
|
||||
expect(await service.runRebalancing()).toBe(false);
|
||||
const failed = service.getRebalanceRun(run.id)!;
|
||||
expect(failed.state).toBe('failed');
|
||||
expect(failed.moves.find(item => item.id === move.id)).toMatchObject({ state: 'failed', message: 'Recovery needed' });
|
||||
expect(failed.moves.filter(item => item.state === 'skipped')).toHaveLength(run.moves.length - 1);
|
||||
});
|
||||
|
||||
it('keeps the source running until a global startup slot is available', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const occupied = beginDeployment('another-server', 'create', 'fixture');
|
||||
await provision();
|
||||
const { hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
expect(hostMigration(move.operationId!)?.remove_operation_id).toBeNull();
|
||||
db().prepare("UPDATE server_deployments SET state='done' WHERE id=?").run(occupied);
|
||||
await provision();
|
||||
expect(hostMigration(move.operationId!)?.remove_operation_id).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('retains its global startup reservation between source removal and destination creation', async () => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing(); await provision();
|
||||
const move = await currentMove(run.id);
|
||||
const { hostMigration } = await import('../../src/lib/server/provisioning/migrations.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(sourceId, hostMigration(move.operationId!)!.remove_operation_id!, { state: 'completed' });
|
||||
await provision();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(move.operationId)).toEqual({ state: 'waiting' });
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const extra = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater',
|
||||
hostId: targetId, libraryId, image, plutoKey: 'another-key' });
|
||||
expect(extra.ok).toBe(true);
|
||||
if (!extra.ok) throw new Error(extra.message);
|
||||
await provision();
|
||||
expect(db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(extra.deploymentId)).toBeUndefined();
|
||||
expect(db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(move.operationId))
|
||||
.toEqual({ operation_id: expect.any(String) });
|
||||
});
|
||||
|
||||
it.each([{ state: 'running', healthy: false }, { state: 'stopped', healthy: false }])('holds removal when fresh inspection returns $state with healthy=$healthy', async health => {
|
||||
const run = await start();
|
||||
const service = await import('../../src/lib/server/rebalancing/index.js');
|
||||
await service.runRebalancing();
|
||||
const move = await currentMove(run.id);
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeRequest).mockResolvedValueOnce(health);
|
||||
await provision();
|
||||
const { hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
expect(hostMigration(move.operationId!)?.remove_operation_id).toBeNull();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT stage FROM server_deployments WHERE id=?').get(move.operationId))
|
||||
.toEqual({ stage: expect.stringContaining('no longer healthy') });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import type { HostInventory } from '../../src/lib/hosts.js';
|
||||
import type { RebalanceSettings } from '../../src/lib/rebalancing.js';
|
||||
|
||||
// Artifact preservation has its own migration tests; these exercise live admission evidence.
|
||||
vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({ captureMoveRecipe: vi.fn(() => ({})) }));
|
||||
const server = 't6_zombie_town_1', gib = 1024 ** 3;
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); });
|
||||
|
||||
async function setup() {
|
||||
const source = await prepareNativeHost([server]), target = await prepareNativeHost();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureRebalancingSchema } = await import('../../src/lib/server/rebalancing/schema.js');
|
||||
ensureRebalancingSchema(db());
|
||||
const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base);
|
||||
const at = new Date().toISOString();
|
||||
const native = db().prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?').get(server) as { id: string; generation: number; library_id: string };
|
||||
const original = db().prepare('SELECT inventory FROM ec_hosts WHERE id=?').get(source.host.id) as { inventory: string };
|
||||
const inventory = JSON.parse(original.inventory) as HostInventory;
|
||||
const resources = { instanceId: native.id, generation: native.generation, cpuCores: 2, memoryBytes: 2 * gib, uptimeSeconds: 600 };
|
||||
db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 75,
|
||||
memoryTotal: 16 * gib, memoryAvailable: 8 * gib, instanceResources: [resources] }), at, source.host.id);
|
||||
db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 0,
|
||||
memoryTotal: 16 * gib, memoryAvailable: 15 * gib, instanceResources: [] }), at, target.host.id);
|
||||
// Target has a verified copy of the source's deployed library version.
|
||||
const cache = db().prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?').get(native.library_id) as { manifest_sha256: string };
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
libraries.updateHostLibraryCaches(target.host.id, [{ libraryVersionId: native.library_id, status: 'ready', manifestSha256: cache.manifest_sha256 }]);
|
||||
db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
|
||||
VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(server, at, at, at);
|
||||
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')").run();
|
||||
db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key')").run(server);
|
||||
const settings: RebalanceSettings = { hostIds: [source.host.id, target.host.id], game: 'all', maxMoves: 10 };
|
||||
const api = await import('../../src/lib/server/rebalancing/snapshot.js');
|
||||
const planner = await import('../../src/lib/server/rebalancing/planner.js');
|
||||
const read = (except = '') => api.rebalanceSnapshot(settings, except);
|
||||
return { db, source, target, native, inventory, resources, settings, read, ...api, ...planner, base, at };
|
||||
}
|
||||
|
||||
it('selects healthy empty native instances on active hosts without creating host work', async () => {
|
||||
const s = await setup();
|
||||
const before = s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get();
|
||||
const snapshot = s.read();
|
||||
expect(snapshot.instances).toHaveLength(1);
|
||||
const plan = s.planRebalance(snapshot, s.settings);
|
||||
expect(plan.moves).toHaveLength(1);
|
||||
expect(plan.moves[0]).toMatchObject({ sourceHostId: s.source.host.id, targetHostId: s.target.host.id });
|
||||
expect(s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get()).toEqual(before);
|
||||
expect(s.rebalanceHosts().every(host => host.eligible)).toBe(true);
|
||||
expect(s.read().hosts.find(host => host.id === s.source.host.id)?.used.cpuCores).toBe(6);
|
||||
});
|
||||
|
||||
it.each(['occupied', 'unknown', 'stopped', 'maintenance', 'recovery', 'lifecycle', 'generation', 'stale-host', 'future-host', 'startup', 'artifacts'] as const)
|
||||
('excludes %s instances with an explanation', async condition => {
|
||||
const s = await setup();
|
||||
if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run();
|
||||
if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run();
|
||||
if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run();
|
||||
if (condition === 'maintenance') s.db().prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)').run(server, s.at);
|
||||
if (condition === 'recovery') s.db().prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, s.at);
|
||||
if (condition === 'lifecycle') s.db().prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at)
|
||||
VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`).run(s.source.host.id, s.native.id, s.at, s.at);
|
||||
if (condition === 'generation') s.db().prepare('UPDATE native_instances SET generation=2').run();
|
||||
if (condition === 'stale-host' || condition === 'future-host') s.db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
|
||||
.run(new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(), s.source.host.id);
|
||||
if (condition === 'startup') s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?").run(s.source.host.id);
|
||||
if (condition === 'artifacts') vi.mocked((await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe).mockImplementationOnce(() => { throw new Error('Deployed files are unavailable.'); });
|
||||
const snapshot = s.read();
|
||||
expect(snapshot.instances).toEqual([]);
|
||||
expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) });
|
||||
});
|
||||
|
||||
it('honors a current positive RCON veto even when telemetry reports empty', async () => {
|
||||
const s = await setup(), status = await import('../../src/lib/server/fleet/status.js');
|
||||
const observe = status.observationReader();
|
||||
vi.spyOn(status, 'observationReader').mockReturnValue(row => ({ ...observe(row), last_players: 0, status_source: 'telemetry' }));
|
||||
s.db().prepare('UPDATE servers SET last_players=1').run();
|
||||
expect(s.read().exclusions[0].reason).toContain('RCON');
|
||||
});
|
||||
|
||||
it('uses conservative recent host pressure and credits only matching observed generation relief', async () => {
|
||||
const s = await setup();
|
||||
s.db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
|
||||
VALUES(?,?,8,90,?,?)`).run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib);
|
||||
s.db().prepare(`INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes)
|
||||
VALUES(?,?,1,'running',?,1,?)`).run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib);
|
||||
const snapshot = s.read();
|
||||
expect(snapshot.hosts.find(host => host.id === s.source.host.id)?.used).toEqual({ cpuCores: 7.2, memoryBytes: 10 * gib });
|
||||
expect(snapshot.instances[0].relief).toEqual({ cpuCores: 1, memoryBytes: gib });
|
||||
expect(snapshot.instances[0].demand).toEqual({ cpuCores: 2, memoryBytes: 2 * gib });
|
||||
});
|
||||
|
||||
it('counts a pending migration once before and after ownership changes and never assumes unplaced pinning', async () => {
|
||||
const s = await setup();
|
||||
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
||||
VALUES('moving',?,'rebuild','waiting','Move',?,'test')`).run(server, s.at);
|
||||
s.db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run('moving', s.native.id);
|
||||
s.db().prepare(`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id)
|
||||
VALUES('moving',?,?,?)`).run(s.source.host.id, s.native.id, s.target.host.id);
|
||||
const target = () => s.read().hosts.find(host => host.id === s.target.host.id)!;
|
||||
expect(target().used.cpuCores).toBe(0.5);
|
||||
expect(target().used.memoryBytes).toBe(2 * gib);
|
||||
expect(s.read('moving').hosts.find(host => host.id === s.target.host.id)?.used.cpuCores).toBe(0);
|
||||
s.db().prepare('UPDATE native_instances SET host_id=?,generation=2').run(s.target.host.id);
|
||||
s.db().prepare('UPDATE native_provisioning SET generation=2').run();
|
||||
s.db().prepare('UPDATE ec_host_migrations SET removed_at=?').run(s.at);
|
||||
expect(target().used.cpuCores).toBe(0.5);
|
||||
expect(target().used.memoryBytes).toBe(2 * gib);
|
||||
s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?")
|
||||
.run(JSON.stringify([{ ...s.resources, generation: 2 }]), 13 * gib, s.target.host.id);
|
||||
expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
|
||||
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
||||
VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`).run(s.at);
|
||||
s.db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run('unplaced', JSON.stringify({ hostId: s.target.host.id }));
|
||||
expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
|
||||
expect(target().instances).toBe(1);
|
||||
});
|
||||
|
||||
it('allows admission to ignore its own unsubmitted rebuild while retaining the source empty evidence', async () => {
|
||||
const s = await setup();
|
||||
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
||||
VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`).run(server, s.at);
|
||||
expect(s.read().instances).toHaveLength(0);
|
||||
expect(s.read('own').instances).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('enforces the 30 minute cooldown across generations after a successful move', async () => {
|
||||
const s = await setup();
|
||||
s.db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`).run(s.at, s.at);
|
||||
s.db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at)
|
||||
VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`).run(server, s.native.id, s.target.host.id, s.source.host.id, s.at);
|
||||
expect(s.read().exclusions[0].reason).toContain('30 minutes');
|
||||
s.db().prepare('UPDATE rebalancing_moves SET updated_at=?').run(new Date(s.base - 30 * 60_000 - 1).toISOString());
|
||||
expect(s.read().instances).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('credits a freed game port only when the current range and bind will actually gain a port', async () => {
|
||||
const s = await setup();
|
||||
expect(s.read().instances[0].releasesPort).toBe(false);
|
||||
s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
|
||||
VALUES(?,?,'0.0.0.0','udp',28960,?)`).run(s.source.host.id, s.native.id, s.at);
|
||||
expect(s.read().instances[0].releasesPort).toBe(true);
|
||||
s.db().prepare('UPDATE ec_hosts SET port_min=28970 WHERE id=?').run(s.source.host.id);
|
||||
expect(s.read().instances[0].releasesPort).toBe(false);
|
||||
s.db().prepare('UPDATE ec_hosts SET port_min=28960 WHERE id=?').run(s.source.host.id);
|
||||
s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
|
||||
VALUES(?,'other','127.0.0.2','tcp',28960,?)`).run(s.source.host.id, s.at);
|
||||
expect(s.read().instances[0].releasesPort).toBe(false);
|
||||
});
|
||||
@@ -45,6 +45,34 @@ it('expires completed file payloads while keeping history, replay identity, hold
|
||||
expect(pruneHistory('operation-payload')).toBe(0);
|
||||
});
|
||||
|
||||
it('retains rebalance recovery files until the move is reconciled, then lets history expire', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
||||
const { pruneHistory } = await import('../../src/lib/server/retention.js');
|
||||
const { putFileBlob } = await import('../../src/lib/server/db/file-blobs.js');
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
ensureHostSchema(db());
|
||||
saveFleetSettings({ ...fleetSettings(), historyKeepCount: 0, provisioningKeepDays: 1, payloadKeepDays: 1 });
|
||||
db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by)
|
||||
VALUES('move','s','rebuild','done','healthy',?,?,'rebalance:run')`).run(old, old);
|
||||
const hash = putFileBlob(Buffer.from('frozen deployed file'));
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)')
|
||||
.run('move', JSON.stringify({ files: [{ path: 'script.gsc', sha256: hash, contentHash: hash }] }));
|
||||
db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message)
|
||||
VALUES('run','{}','running',?,?,'waiting')`).run(old, old);
|
||||
db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,operation_id,message,updated_at)
|
||||
VALUES('m','run',0,'s','i',1,'a','b','{}','moving','move','waiting',?)`).run(old);
|
||||
expect(pruneHistory('workflow-payload')).toBe(0);
|
||||
expect(pruneHistory('provisioning')).toBe(0);
|
||||
db().prepare("UPDATE rebalancing_moves SET state='failed'").run();
|
||||
expect(pruneHistory('workflow-payload')).toBe(0);
|
||||
expect(pruneHistory('provisioning')).toBe(0);
|
||||
db().prepare("UPDATE rebalancing_moves SET state='done'").run();
|
||||
expect(pruneHistory('workflow-payload')).toBe(1);
|
||||
expect(pruneHistory('provisioning')).toBe(1);
|
||||
expect(db().prepare('SELECT state,operation_id FROM rebalancing_moves').get()).toEqual({ state: 'done', operation_id: null });
|
||||
});
|
||||
|
||||
it('normalizes legacy retention clocks once and reuses clock indexes', () => {
|
||||
const handle = new Database(':memory:');
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { expect, it } from 'vitest';
|
||||
import { render } from 'svelte/server';
|
||||
import type { RebalanceCandidate, RebalancePreview, RebalanceRun } from '../../src/lib/rebalancing.js';
|
||||
import RebalancePreviewView from '../../src/lib/ui/components/capacity/RebalancePreviewView.svelte';
|
||||
import RebalanceRunView from '../../src/lib/ui/components/capacity/RebalanceRunView.svelte';
|
||||
|
||||
const at = '2026-10-08T12:00:00Z';
|
||||
const move: RebalanceCandidate = { server: 't5_zombie_theater_1', instanceId: 'instance-a', generation: 1,
|
||||
sourceHostId: 'source-a', sourceHostName: 'Source A', targetHostId: 'target-b', targetHostName: 'Target B', improvement: 0.1 };
|
||||
const preview: RebalancePreview = { id: 'preview-a', createdAt: at, expiresAt: '2026-10-08T12:05:00Z',
|
||||
settings: { hostIds: ['source-a', 'target-b'], game: 'all', maxMoves: 5 }, moves: [move],
|
||||
hosts: [{ id: 'source-a', name: 'Source A', before: { cpuPercent: 75, memoryPercent: 50, instances: 6 }, after: { cpuPercent: 62.5, memoryPercent: 42, instances: 5 } }],
|
||||
exclusions: [{ server: 't6_zm_transit_1', reason: 'Lobby has players.' }], warnings: ['Occupied lobbies stay in place.'] };
|
||||
const run: RebalanceRun = { id: 'run-a', createdAt: at, updatedAt: at, settings: preview.settings, state: 'running', message: 'Moving one lobby.',
|
||||
moves: [{ ...move, id: 'move-a', state: 'moving', operationId: 'operation-a', message: 'Waiting for destination health.', updatedAt: at }] };
|
||||
const noop = () => {};
|
||||
|
||||
it('shows current and projected resources, the move destination, and exclusion reasons before starting', () => {
|
||||
const body = render(RebalancePreviewView, { props: { preview, expired: false, onstart: noop } }).body;
|
||||
expect(body).toContain('6 → 5');
|
||||
expect(body).toContain('75% → 62.5%');
|
||||
expect(body).toContain('50% → 42%');
|
||||
expect(body).toContain('Source A → Target B');
|
||||
expect(body).toContain('Lobby has players.');
|
||||
expect(body).toContain('Occupied lobbies stay in place.');
|
||||
expect(body).toContain('aria-label="Projected host distribution" tabindex="0"');
|
||||
expect(body).not.toMatch(/<button[^>]*disabled/);
|
||||
});
|
||||
|
||||
it.each(['expired', 'empty', 'blocked'] as const)('prevents starting a %s preview', condition => {
|
||||
const body = render(RebalancePreviewView, { props: {
|
||||
preview: condition === 'empty' ? { ...preview, moves: [] } : preview,
|
||||
expired: condition === 'expired', disabled: condition === 'blocked', onstart: noop
|
||||
} }).body;
|
||||
expect(body).toMatch(/<button[^>]*disabled/);
|
||||
if (condition === 'expired') expect(body).toContain('This preview has expired.');
|
||||
if (condition === 'empty') expect(body).toContain('No eligible moves would improve');
|
||||
});
|
||||
|
||||
it('keeps run progress tied to its provisioning record and explains stopping semantics', () => {
|
||||
const body = render(RebalanceRunView, { props: { run, canManage: true, onstop: noop } }).body;
|
||||
expect(body).toContain('0 of 1 moves completed');
|
||||
expect(body).toContain('Waiting for destination health.');
|
||||
expect(body).toContain('href="/provisioning/operation-a?returnTo=%2Fcapacity%3Frebalance%3D1%26rebalanceRun%3Drun-a"');
|
||||
expect(body).toContain('Stop remaining moves');
|
||||
expect(body).toContain('A move already in progress continues through health verification.');
|
||||
});
|
||||
|
||||
it('hides stop from viewers and terminal or already-stopping runs', () => {
|
||||
expect(render(RebalanceRunView, { props: { run, canManage: false, onstop: noop } }).body).not.toContain('Stop remaining moves');
|
||||
for (const state of ['stopping', 'done', 'stopped', 'failed'] as const) {
|
||||
const body = render(RebalanceRunView, { props: { run: { ...run, state }, canManage: true, onstop: noop } }).body;
|
||||
expect(body).not.toContain('Stop remaining moves');
|
||||
expect(body).toContain('View provisioning');
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user