import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_test']); }); afterEach(async () => { await fixture.cleanup(); }); it('counts rebuilds once across pools and keeps their capacity through migration handoff and lower limits', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { concurrencyBlocker, lifecycleHolders, preparingJobs } = await import('../../src/lib/server/provisioning/concurrency.js'); const h = db(); h.prepare( "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','source',1,'lib','img',1,'2026-01-01')" ).run(); h.prepare( "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','rebuild','waiting','migrating','2026-01-01','test')" ).run(); h.prepare( "INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)" ).run(); h.prepare( "INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id,removed_at) VALUES('a','source','i','target','remove','2026-01-02')" ).run(); saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 1, rebuildConcurrency: 1 }); expect(lifecycleHolders()).toHaveLength(1); expect(concurrencyBlocker('b', 'stop', 'other')).toContain('lifecycle'); expect(concurrencyBlocker('a', 'rebuild', 'target')).toBeNull(); saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 0 }); expect(concurrencyBlocker('b', 'rebuild', 'other')).toContain('rebuild'); saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 0, createConcurrencyPerHost: 1 }); expect(concurrencyBlocker('b', 'create', 'target')).toContain('startup'); expect(concurrencyBlocker('b', 'create', 'other')).toBeNull(); preparingJobs.add('a'); expect(lifecycleHolders()).toHaveLength(1); preparingJobs.clear(); }); it('enforces per-host removal and recovery limits independently of other hosts', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { concurrencyBlocker } = await import('../../src/lib/server/provisioning/concurrency.js'); const h = db(); h.prepare("INSERT INTO servers(name) VALUES('t5_test')").run(); h.prepare( "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','host',1,'lib','img',1,'2026-01-01')" ).run(); h.prepare( "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','destroy','running','removing','2026-01-01','test')" ).run(); h.prepare( "INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)" ).run(); saveFleetSettings({ ...fleetSettings(), destroyConcurrencyPerHost: 1, restartConcurrencyPerHost: 1 }); expect(concurrencyBlocker('b', 'destroy', 'host')).toContain('destruction'); expect(concurrencyBlocker('b', 'destroy', 'other')).toBeNull(); h.prepare( "INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('t5_test','2026-01-01','a')" ).run(); expect(concurrencyBlocker('b', 'restart', 'host')).toContain('recovery'); expect(concurrencyBlocker('b', 'restart', 'other')).toBeNull(); }); it('counts a creation recovery rebuild against rebuild capacity without losing its startup reservation', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { concurrencyBlocker, lifecycleHolders } = await import('../../src/lib/server/provisioning/concurrency.js'); db() .prepare( "INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','t5_test','host','lib','img',1,'2026-01-01')" ) .run(); db() .prepare( "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','create','running','recovery','2026-01-01','test')" ) .run(); db() .prepare( "INSERT INTO native_provisioning(deployment_id,instance_id,generation,recovery_step) VALUES('a','i',1,'rebuild')" ) .run(); saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 1, createConcurrencyPerHost: 1 }); expect(lifecycleHolders()).toEqual([expect.objectContaining({ id: 'a', action: 'rebuild' })]); expect(concurrencyBlocker('b', 'rebuild', 'other')).toContain('rebuild'); expect(concurrencyBlocker('b', 'create', 'host')).toContain('startup'); expect(concurrencyBlocker('a', 'rebuild', 'host')).toBeNull(); }); it('retains every capacity owner across failure, cancellation and migration without counting completed history', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { createHost } = await import('../../src/lib/server/hosts/index.js'); const { lifecycleHolders, preparingJobs } = await import('../../src/lib/server/provisioning/concurrency.js'); const h = db(), host = createHost({ name: 'Source' }); const add = (id: string, state: string, command?: string, migration = false) => { h.prepare( "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,?,'rebuild',?,'test','2026-01-01','test')" ).run(id, id, state); h.prepare( "INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'lib','img',1,'2026-01-01')" ).run(id, id, host.id); h.prepare( 'INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,?,1,?)' ).run(id, id, migration ? null : command ? id : null); if (command) h.prepare( "INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.rebuild',?,'{}','2026-01-01','2026-01-01')" ).run(id, host.id, command); if (migration) h.prepare( 'INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id) VALUES(?,?,?,?,?)' ).run(id, host.id, id, 'target', command ? id : null); }; add('running', 'running', 'running'); add('health', 'awaiting-health', 'completed'); add('prepared', 'queued'); add('recovery', 'failed', 'failed'); add('cancelled-command', 'cancelled', 'unknown'); add('done-command', 'done', 'pending'); add('failed-migration', 'failed', 'completed', true); add('unknown-migration', 'legacy-unknown', 'completed', true); add('cancelled-migration', 'cancelled', 'accepted', true); add('waiting-migration', 'waiting', 'running', true); add('finished', 'done', 'completed'); add('cancelled-finished', 'cancelled', 'completed', true); add('finished-migration', 'warning', 'completed', true); add('queued', 'queued'); h.prepare("INSERT INTO servers(name) VALUES('recovery')").run(); h.prepare( "INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('recovery','recovery','2026-01-01')" ).run(); preparingJobs.add('prepared'); try { const holders = lifecycleHolders(); expect(holders.map((row) => row.id).sort()).toEqual( [ 'running', 'health', 'prepared', 'recovery', 'cancelled-command', 'done-command', 'failed-migration', 'unknown-migration', 'cancelled-migration', 'waiting-migration' ].sort() ); expect(holders.find((row) => row.id === 'waiting-migration')).toMatchObject({ host_id: host.id, target_host_id: 'target', removing: 1 }); } finally { preparingJobs.clear(); } }); it('seeks capacity owners through active indexes instead of scanning retained jobs and receipts', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { createHost } = await import('../../src/lib/server/hosts/index.js'); const { lifecycleHolders } = await import('../../src/lib/server/provisioning/concurrency.js'); const h = db(), host = createHost({ name: 'History host' }); h.transaction(() => { const job = h.prepare( "INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'old','restart','done','Done','2026-01-01','test')" ); const operation = h.prepare( "INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.restart','completed','{}','2026-01-01','2026-01-01')" ); h.prepare( "INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','old',?,'lib','img',1,'2026-01-01')" ).run(host.id); const native = h.prepare( "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,'i',1,?)" ); const migration = h.prepare( "INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id,removed_at) VALUES(?,?,'i',?,'2026-01-01')" ); for (let n = 0; n < 4000; n++) { const id = String(n); job.run(id); operation.run(id, host.id); native.run(id, id); migration.run(id, host.id, id); } })(); const prepare = vi.spyOn(h, 'prepare'); let sql: string; try { expect(lifecycleHolders()).toEqual([]); sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT d.id,'))![0]; } finally { prepare.mockRestore(); } const plan = h.prepare(`EXPLAIN QUERY PLAN ${sql}`).all('[]') as { detail: string }[]; const details = plan.map((row) => row.detail).join('\n'); for (const index of [ 'idx_server_deployments_state', 'idx_server_deployments_unsettled', 'ec_host_operations_active', 'native_provisioning_operation', 'ec_host_migrations_operation' ]) { expect(details).toContain(index); } expect(details).not.toMatch(/SCAN (?:d|n|m|o)\b/); // Scanning the small partial index of unsettled IDs is intentional. expect(details).not.toMatch( /SCAN server_deployments(?! USING INDEX idx_server_deployments_unsettled)\b/ ); });