josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add fleet and per-host concurrency controls

Browse Source
This commit is contained in:
josh
2026-10-05 22:27:27 -04:00
parent c3b990c5fd
commit 7c98df4fb0
14 changed files with 268 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/concurrency.test.ts
+45
View File
@@ -0,0 +1,45 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_test']); });
afterEach(async () => { await fixture.cleanup(); });
it('counts rebuilds once across pools and keeps their capacity through migration handoff and lower limits', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { concurrencyBlocker, lifecycleHolders, preparingJobs } = await import('../../src/lib/server/provisioning/concurrency.js');
const h = db();
h.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','source',1,'lib','img',1,'2026-01-01')").run();
h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','rebuild','waiting','migrating','2026-01-01','test')").run();
h.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)").run();
h.prepare("INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id,removed_at) VALUES('a','source','i','target','remove','2026-01-02')").run();
saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 1, rebuildConcurrency: 1 });
expect(lifecycleHolders()).toHaveLength(1);
expect(concurrencyBlocker('b','stop','other')).toContain('lifecycle');
expect(concurrencyBlocker('a','rebuild','target')).toBeNull();
saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 0 });
expect(concurrencyBlocker('b','rebuild','other')).toContain('rebuild');
saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 0, createConcurrencyPerHost: 1 });
expect(concurrencyBlocker('b','create','target')).toContain('startup');
expect(concurrencyBlocker('b','create','other')).toBeNull();
preparingJobs.add('a');
expect(lifecycleHolders()).toHaveLength(1);
preparingJobs.clear();
});
it('enforces per-host removal and recovery limits independently of other hosts', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { concurrencyBlocker } = await import('../../src/lib/server/provisioning/concurrency.js');
const h = db();
h.prepare("INSERT INTO servers(name) VALUES('t5_test')").run();
h.prepare("INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','host',1,'lib','img',1,'2026-01-01')").run();
h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','destroy','running','removing','2026-01-01','test')").run();
h.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)").run();
saveFleetSettings({ ...fleetSettings(), destroyConcurrencyPerHost: 1, restartConcurrencyPerHost: 1 });
expect(concurrencyBlocker('b','destroy','host')).toContain('destruction');
expect(concurrencyBlocker('b','destroy','other')).toBeNull();
h.prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('t5_test','2026-01-01','a')").run();
expect(concurrencyBlocker('b','restart','host')).toContain('recovery');
expect(concurrencyBlocker('b','restart','other')).toBeNull();
});
tests/unit/fleet-settings.test.ts
+17 -5
View File
@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const defaults = { deployKeepDays: 7, observationKeepDays: 1, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
import { defaultFleetSettings as defaults } from '../../src/lib/fleet-settings.js';
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
@@ -14,7 +14,7 @@ describe('fleet settings API and persistence', () => {
it('returns defaults and saves all settings across a module reload and database reopen', async () => {
const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
expect(await GET().json()).toEqual(defaults);
const saved = { deployKeepDays: 30, observationKeepDays: 5, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
const saved = { ...defaults, deployKeepDays: 30, observationKeepDays: 5, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
expect(response.status).toBe(200);
expect(await response.json()).toEqual(saved);
@@ -37,7 +37,7 @@ describe('fleet settings API and persistence', () => {
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
const { fleetSettings } = await import('../../src/lib/server/settings.js');
expect(fleetSettings()).toEqual({ deployKeepDays: 14, observationKeepDays: 1, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
expect(fleetSettings()).toEqual({ ...defaults, deployKeepDays: 14, observationKeepDays: 1, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
expect(fleetSettings().deployKeepDays).toBe(14);
@@ -53,7 +53,7 @@ describe('fleet settings API and persistence', () => {
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '9');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '8');
const { GET } = await import('../../src/routes/api/settings/+server.js');
const upgraded = { ...previous, destroyConcurrency: 1, observationKeepDays: 1 };
const upgraded = { ...defaults, ...previous, destroyConcurrency: 1, observationKeepDays: 1 };
expect(await GET().json()).toEqual(upgraded);
expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
closeDb();
@@ -68,7 +68,7 @@ describe('fleet settings API and persistence', () => {
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', 'bad');
const { GET } = await import('../../src/routes/api/settings/+server.js');
const upgraded = { ...previous, observationKeepDays: 1 };
const upgraded = { ...defaults, ...previous, observationKeepDays: 1 };
expect(await GET().json()).toEqual(upgraded);
expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
expect(db().prepare("SELECT value FROM settings WHERE key='unrelated'").get()).toEqual({ value: 'keep' });
@@ -125,3 +125,15 @@ it('applies saved retention to cleanup while protecting unfinished campaigns and
expect(db().prepare("SELECT server FROM deployment_servers WHERE deployment_id = 'completed'").all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([{ id: 'history' }]);
});
it('saves a section without overwriting changes in the other section', async () => {
const { PATCH, GET } = await import('../../src/routes/api/settings/+server.js');
const { concurrencyKeys } = await import('../../src/lib/fleet-settings.js');
const { saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...defaults, deployKeepDays: 42 });
const response = await PATCH({ request: new Request('http://localhost/api/settings', { method: 'PATCH', body: JSON.stringify({
section: 'Concurrency', values: Object.fromEntries(concurrencyKeys.map(key => [key, key === 'rebuildConcurrency' ? 2 : defaults[key]]))
}) }) });
expect(response.status).toBe(200);
expect(await GET().json()).toMatchObject({ deployKeepDays: 42, rebuildConcurrency: 2 });
});
tests/unit/native-hosts.test.ts
+17
View File
@@ -353,3 +353,20 @@ describe('durable host operations', () => {
expect(api.getHostOperation(operation.id)?.state).toBe('pending');
});
});
it('limits preparing hosts at delivery, preserves replays, and releases capacity after completion', async () => {
const api = await enrolled();
const second = api.createHost({ name: 'Second' });
api.enrollHost(api.createEnrollment(second.id).token, inventory);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), preparationConcurrency: 1 });
const first = api.queueHostOperation(api.host.id, 'library.download', { libraryVersionId: 'one' });
const next = api.queueHostOperation(second.id, 'library.download', { libraryVersionId: 'two' });
const inspect = api.queueHostOperation(second.id, 'host.inspect', {});
expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id);
expect(api.pendingHostOperations(second.id).map(row => row.id)).not.toContain(next.id);
expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(inspect.id);
expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(first.id);
api.reportHostOperation(api.host.id, first.id, { state: 'completed', result: {} });
expect(api.pendingHostOperations(second.id).map(row => row.id)).toContain(next.id);
});
tests/unit/native-provisioning.test.ts
+18
View File
@@ -1372,3 +1372,21 @@ describe('cancelled native mutation bookkeeping', () => {
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
});
});
describe('additional concurrency admission', () => {
it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)('holds simultaneous startups with %s until recovery completes', async key => {
const first = await queue();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 });
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
if (!second.ok) throw new Error(second.message);
await advance();
const operations = [await operationFor(first.deploymentId), await operationFor(second.deploymentId)];
expect(operations.filter(Boolean)).toHaveLength(1);
expect((await detail(second.deploymentId)).stage).toContain('capacity');
saveFleetSettings({ ...fleetSettings(), [key]: 0 });
await advance();
expect(await operationFor(second.deploymentId)).not.toBeNull();
});
});