Add safe manual instance rebalancing
This commit is contained in:
@@ -11,6 +11,9 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
|
||||
rebalanceAdmission: vi.fn(() => null), requestRebalanceStopForOperation: vi.fn()
|
||||
}));
|
||||
|
||||
let queuedKeys = 0;
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
@@ -84,6 +87,123 @@ async function observations(server: string) {
|
||||
return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
|
||||
}
|
||||
|
||||
async function moveTarget() {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
const target = hosts.createHost({ name: 'Rebalance destination', portMin: 29000, portMax: 29005 });
|
||||
hosts.enrollHost(hosts.createEnrollment(target.id).token, inventory());
|
||||
libraries.updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready',
|
||||
manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]);
|
||||
return target;
|
||||
}
|
||||
|
||||
describe('rebalance instance migrations', () => {
|
||||
it('preserves deployed configuration, patched file versions, identity and credentials on an active source', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
Object.assign(config.rcon, { password: '' });
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const baseline = createScript({ name: 'Deployed baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5',
|
||||
content: Buffer.from('set sv_voice "1"\n') });
|
||||
const script = createScript({ name: 'Deployed script', kind: 'script', targetRelPath: 'preserved.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('original deployed script') });
|
||||
for (const artifact of [baseline, script]) db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id);
|
||||
const job = await queue(); await advance();
|
||||
const launch = (await operationFor(job.deploymentId))!.payload as { rconPassword: string; files: { path: string; sha256: string }[] };
|
||||
await acknowledge(job.deploymentId);
|
||||
const patched = uploadNewVersion(script.script.id, Buffer.from('confirmed patch after initial launch'));
|
||||
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
recordScript(job.name, 'preserved.gsc', patched.id, patched.sha256, patched.size, 'campaign');
|
||||
const newest = uploadNewVersion(script.script.id, Buffer.from('new promotion must not be installed'));
|
||||
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newest.id, script.script.id);
|
||||
const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n'));
|
||||
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newBaseline.id, baseline.script.id);
|
||||
Object.assign(config.rcon, { password: 'new-controller-password' });
|
||||
const target = await moveTarget();
|
||||
const { captureMoveRecipe, requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const original = nativeInstance(job.name)!;
|
||||
const recipe = captureMoveRecipe(job.name);
|
||||
expect(recipe.files.every(file => file.content === undefined)).toBe(true);
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test', recipe });
|
||||
await advance();
|
||||
const removal = hostMigration(id)!.remove_operation_id!;
|
||||
expect(removal).toBeTruthy();
|
||||
const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(canCancelProvisioningJob(id)).toBe(false);
|
||||
expect(cancelProvisioningJob(id)).toBe(false);
|
||||
const { getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
||||
expect(getDeployment(id)?.canCancel).toBe(false);
|
||||
expect(listDeployments().find(item => item.id === id)?.canCancel).toBe(0);
|
||||
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
|
||||
const response = POST({ params: { id } });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ error: expect.stringContaining('must finish recreating') });
|
||||
const { reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, removal, { state: 'completed' });
|
||||
await advance(); await advance();
|
||||
const creation = (await operationFor(id))!;
|
||||
expect(creation).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2,
|
||||
payload: { name: job.name, libraryId, image, rconPassword: launch.rconPassword, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
|
||||
const payload = creation.payload as { files: { path: string; sha256: string }[] };
|
||||
expect(payload.files).toContainEqual(expect.objectContaining({ path: expect.stringContaining('preserved.gsc'), sha256: patched.sha256 }));
|
||||
expect(payload.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256)
|
||||
.toBe(launch.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256);
|
||||
reportHostOperation(target.id, creation.id, { state: 'completed', result: { state: 'running', healthy: true } });
|
||||
await advance();
|
||||
expect((await detail(id)).state).toBe('done');
|
||||
expect(nativeInstance(job.name)).toMatchObject({ id: original.id, host_id: target.id, generation: 2 });
|
||||
expect(getHost(hostId)?.mode).toBe('active');
|
||||
expect(JSON.stringify(getDeployment(id)?.recipe)).not.toContain(launch.rconPassword);
|
||||
});
|
||||
|
||||
it('rechecks occupancy, target capacity and run admission before removing the source', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const target = await moveTarget();
|
||||
const { requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
expect(() => requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' })).toThrow('different destination');
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
||||
remote.players = 2; await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
remote.players = 0;
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(target.id, { ...inventory(), memoryAvailable: 1 });
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
heartbeat(target.id, inventory());
|
||||
const { rebalanceAdmission } = await import('../../src/lib/server/rebalancing/admission.js');
|
||||
vi.mocked(rebalanceAdmission).mockReturnValueOnce('Run stopped during occupancy check.');
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toBeNull();
|
||||
await advance();
|
||||
expect(hostMigration(id)?.remove_operation_id).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('withdraws only uncommitted moves and preserves their server keys', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const target = await moveTarget();
|
||||
const { requestInstanceMove } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
const { requestRebalanceStopForOperation } = await import('../../src/lib/server/rebalancing/admission.js');
|
||||
expect(requestRebalanceStopForOperation).toHaveBeenCalledWith(id);
|
||||
await advance();
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(job.name)).toBe('saved-key');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses to move when recorded deployed bytes cannot be restored', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { captureMoveRecipe } = await import('../../src/lib/server/provisioning/host-drain.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE server_scripts SET sha256=? WHERE server=?").run('0'.repeat(64), job.name);
|
||||
expect(() => captureMoveRecipe(job.name)).toThrow('deployed bytes are unavailable');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runtime configuration at dispatch', () => {
|
||||
it.each([
|
||||
['t5', 'shared'], ['t5', 'generated'], ['t6', 'shared'], ['t6', 'generated']
|
||||
|
||||
Reference in New Issue
Block a user