josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Move fleet retention and concurrency controls into Settings

Browse Source 
Persist settings in SQLite, import legacy environment values once, and apply changes to cleanup and deployment queues without restarting.

Closes #51
This commit is contained in:
josh
2026-09-27 16:11:45 -04:00
parent fc46cb9da4
commit 4fe8613678
20 changed files with 334 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/creation-queue.test.ts
+42 -3
View File
@@ -66,6 +66,44 @@ async function states() {
}
describe('shared creation queue', () => {
it('rechecks a lowered limit after awaiting a startup in the same pass', async () => {
await queueServers(3);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => {
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
return { ok: true };
});
await processCreations();
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
});
it('applies live increases and decreases while preserving active startups', async () => {
const jobs = await queueServers(4);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const { komodoInspectHealthy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'still starting' });
await processCreations();
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
await processCreations();
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']);
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
await processCreations();
expect(komodoCreateDeployment).toHaveBeenCalledTimes(3);
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']);
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[2].name
? { ok: false, reason: 'still starting' } : { ok: true, state: 'running' });
await processCreations();
expect(await states()).toEqual(['done', 'done', 'awaiting-health', 'queued']);
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
await processCreations();
expect(await states()).toEqual(['done', 'done', 'done', 'awaiting-health']);
});
it('starts only one server per pass and holds its slot until a later health check', async () => {
const jobs = await queueServers(3);
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
@@ -345,13 +383,14 @@ describe('shared creation queue', () => {
describe('creation concurrency configuration', () => {
it('defaults to one startup', async () => {
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined);
const { config } = await import('../../src/lib/server/config.js');
expect(config.createConcurrency).toBe(1);
const { fleetSettings } = await import('../../src/lib/server/settings.js');
expect(fleetSettings().createConcurrency).toBe(1);
});
it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => {
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value);
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_CREATE_CONCURRENCY/);
const { fleetSettings } = await import('../../src/lib/server/settings.js');
expect(() => fleetSettings()).toThrow(/Concurrent server startups/);
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
});
});
tests/unit/deployment-collections.test.ts
+10
View File
@@ -51,6 +51,16 @@ function keysFor(preview: CollectionPreview) {
}
describe('multi-server blueprints', () => {
it('shows the saved startup limit in new previews and existing collection progress', async () => {
const { blueprint, preview } = await previewLineup();
const { queueCollection, getCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js');
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
expect(previewCollection(blueprint.id).launchConcurrency).toBe(3);
expect(getCollection(result.collectionId!)?.queue.concurrency).toBe(3);
});
it('saves a mixed-game lineup and previews unique slots and global ports without reserving them', async () => {
const { blueprint, preview } = await previewLineup();
const { db } = await import('../../src/lib/server/db/index.js');
tests/unit/fleet-settings.test.ts
+81
View File
@@ -0,0 +1,81 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 };
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
for (const key of ['DEPLOY_KEEP_DAYS', 'DEPLOYMENT_RESTART_CONCURRENCY', 'DEPLOYMENT_CREATE_CONCURRENCY']) vi.stubEnv(key, undefined);
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
describe('fleet settings API and persistence', () => {
it('returns defaults and saves all settings across a module reload and database reopen', async () => {
const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
expect(await GET().json()).toEqual(defaults);
const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2 };
const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
expect(response.status).toBe(200);
expect(await response.json()).toEqual(saved);
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb(); vi.resetModules();
// Once saved, even invalid legacy environment values are irrelevant.
vi.stubEnv('DEPLOY_KEEP_DAYS', '-1');
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', 'bad');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '9');
const reopened = await import('../../src/routes/api/settings/+server.js');
expect(await reopened.GET().json()).toEqual(saved);
});
it('imports legacy values into an existing database once and preserves unrelated settings', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("DELETE FROM settings WHERE key = 'fleet_settings'").run();
db().prepare("INSERT INTO settings (key, value) VALUES ('unrelated', 'true')").run();
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', '14');
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
const { fleetSettings } = await import('../../src/lib/server/settings.js');
expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2 });
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
expect(fleetSettings().deployKeepDays).toBe(14);
expect(db().prepare("SELECT value FROM settings WHERE key = 'unrelated'").get()).toEqual({ value: 'true' });
});
it.each([
'null', '[]', '{}', '{',
...['deployKeepDays', 'restartConcurrency', 'createConcurrency'].flatMap(key =>
[0, -1, 1.5, '2', true, null, 9007199254740992].map(value => JSON.stringify({ ...defaults, [key]: value }))),
JSON.stringify({ ...defaults, deployKeepDays: 36501 }),
JSON.stringify({ ...defaults, unknown: 1 })
])('rejects invalid input without partially saving: %s', async body => {
const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body }) });
expect(response.status).toBe(400);
expect(await response.json()).toHaveProperty('error');
expect(await GET().json()).toEqual(defaults);
});
});
it('applies saved retention to cleanup while protecting unfinished campaigns and provisioning history', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const old = new Date(Date.now() - 10 * 86400000).toISOString();
for (const id of ['completed', 'pending', 'failed', 'held']) {
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES (?, 'test', ?)").run(id, old);
db().prepare('INSERT INTO deployment_servers (deployment_id, server, restart_state, files_changed) VALUES (?, ?, ?, 0)')
.run(id, 'server-' + id, id === 'completed' ? 'done' : id);
}
db().prepare("INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('history', 'removed', 'destroy', 'done', 'Complete', ?, 'ui')").run(old);
saveFleetSettings({ ...fleetSettings(), deployKeepDays: 30 });
await sweep();
expect(db().prepare('SELECT id FROM deployments').all()).toHaveLength(4);
saveFleetSettings({ ...fleetSettings(), deployKeepDays: 7 });
await sweep();
expect(db().prepare('SELECT id FROM deployments ORDER BY id').all()).toEqual([{ id: 'failed' }, { id: 'held' }, { id: 'pending' }]);
expect(db().prepare("SELECT server FROM deployment_servers WHERE deployment_id = 'completed'").all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([{ id: 'history' }]);
});
tests/unit/restart-rollout.test.ts
+17 -12
View File
@@ -438,10 +438,8 @@ describe('Provisioning ownership', () => {
describe('rollout configuration', () => {
it('honors a configured limit larger than one', async () => {
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
vi.resetModules();
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
await deployment();
const { sweep } = await import('../helpers/provisioning.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
@@ -451,13 +449,20 @@ describe('rollout configuration', () => {
expect(komodoRestart).toHaveBeenCalledTimes(2);
});
it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => {
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
vi.resetModules();
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value);
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/);
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
vi.resetModules();
it('holds new restarts after lowering a live limit without releasing active slots', async () => {
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { sweep } = await import('../helpers/provisioning.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
await deployment();
await sweep();
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
await sweep();
expect(komodoRestart).toHaveBeenCalledTimes(2);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all()).toHaveLength(2);
nextTick();
await sweep();
expect(komodoRestart).toHaveBeenCalledTimes(3);
});
});