diff --git a/src/lib/server/backends/schema.ts b/src/lib/server/backends/schema.ts index 41c1688..506debf 100644 --- a/src/lib/server/backends/schema.ts +++ b/src/lib/server/backends/schema.ts @@ -23,6 +23,7 @@ export function migrateNativeInstances(handle: Database.Database): void { applied_at TEXT, verified_at TEXT ); CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id); + CREATE INDEX IF NOT EXISTS native_provisioning_operation ON native_provisioning(operation_id, deployment_id); CREATE TABLE IF NOT EXISTS provisioning_safety_overrides ( deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE, actor_id TEXT NOT NULL, actor_name TEXT NOT NULL, confirmed_at TEXT NOT NULL @@ -35,6 +36,7 @@ export function migrateNativeInstances(handle: Database.Database): void { ); CREATE INDEX IF NOT EXISTS ec_host_migrations_source ON ec_host_migrations(source_host_id); CREATE INDEX IF NOT EXISTS ec_host_migrations_target ON ec_host_migrations(target_host_id); + CREATE INDEX IF NOT EXISTS ec_host_migrations_operation ON ec_host_migrations(remove_operation_id, deployment_id); CREATE TABLE IF NOT EXISTS native_creation_cancellations ( creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id), cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id) diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 812fac3..9cfe614 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -393,12 +393,16 @@ export function db(): Database.Database { ON server_deployments(server, server_deleted_at); CREATE INDEX IF NOT EXISTS idx_server_deployments_state ON server_deployments(state, action, server_deleted_at); + CREATE INDEX IF NOT EXISTS idx_server_deployments_server_state + ON server_deployments(server, server_deleted_at, state, action); CREATE INDEX IF NOT EXISTS idx_server_deployments_started_at ON server_deployments(started_at); CREATE INDEX IF NOT EXISTS idx_server_deployments_submission ON server_deployments(server, server_deleted_at, action, submitted_at); CREATE INDEX IF NOT EXISTS idx_server_deployments_finished_at - ON server_deployments(finished_at) WHERE server_deleted_at IS NULL;`); + ON server_deployments(finished_at) WHERE server_deleted_at IS NULL; + CREATE INDEX IF NOT EXISTS idx_server_deployments_unsettled + ON server_deployments(id) WHERE state NOT IN ('done','warning','cancelled');`); installReportRevisions(handle); } catch (err) { handle.close(); diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index af4ed3e..37ec541 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -44,6 +44,8 @@ export function ensureHostSchema(handle: Database.Database): void { -- causing a history scan for every queued command on Node's main thread. CREATE INDEX IF NOT EXISTS ec_host_operations_kind ON ec_host_operations(host_id, kind, created_at); CREATE INDEX IF NOT EXISTS ec_host_operations_kind_state ON ec_host_operations(kind, state); + CREATE INDEX IF NOT EXISTS ec_host_operations_active ON ec_host_operations(state, id) + WHERE state IN ('pending','accepted','running','unknown'); CREATE TABLE IF NOT EXISTS ec_host_operation_payloads ( operation_id TEXT PRIMARY KEY REFERENCES ec_host_operations(id) ON DELETE CASCADE, payload TEXT NOT NULL diff --git a/src/lib/server/provisioning/concurrency.ts b/src/lib/server/provisioning/concurrency.ts index 32838e9..b0e6adf 100644 --- a/src/lib/server/provisioning/concurrency.ts +++ b/src/lib/server/provisioning/concurrency.ts @@ -8,18 +8,29 @@ interface Holder { id: string; action: string; host_id: string | null; target_ho export function lifecycleHolders(): Holder[] { ensureHostSchema(db()); + // Find capacity owners through their active state or outstanding command. + // A correlated OR over d used to walk every retained job and fetch its agent + // receipt on every admission, even when all that history was completed. return db().prepare(`SELECT d.id,d.action,i.host_id,m.target_host_id, CASE WHEN m.remove_operation_id IS NOT NULL AND m.removed_at IS NULL THEN 1 ELSE 0 END AS removing FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id LEFT JOIN native_instances i ON i.id=n.instance_id LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id - WHERE d.state IN ('running','awaiting-health') - OR d.id IN (SELECT value FROM json_each(?)) - OR EXISTS (SELECT 1 FROM restart_rollout r WHERE r.operation_id=d.id AND r.healthy_at IS NULL) - OR (m.remove_operation_id IS NOT NULL AND d.state NOT IN ('done','warning','cancelled')) - OR EXISTS (SELECT 1 FROM ec_host_operations o WHERE o.id IN (n.operation_id,m.remove_operation_id) - AND o.state IN ('pending','accepted','running','unknown'))`) + WHERE d.id IN ( + SELECT id FROM server_deployments WHERE state IN ('running','awaiting-health') + UNION SELECT value FROM json_each(?) + UNION SELECT operation_id FROM restart_rollout WHERE healthy_at IS NULL + UNION SELECT deployment_id FROM ec_host_migrations + WHERE remove_operation_id IS NOT NULL AND deployment_id IN ( + SELECT id FROM server_deployments WHERE state NOT IN ('done','warning','cancelled')) + UNION SELECT n.deployment_id FROM ec_host_operations o + JOIN native_provisioning n ON n.operation_id=o.id + WHERE o.state IN ('pending','accepted','running','unknown') + UNION SELECT m.deployment_id FROM ec_host_operations o + JOIN ec_host_migrations m ON m.remove_operation_id=o.id + WHERE o.state IN ('pending','accepted','running','unknown') + )`) .all(JSON.stringify([...preparingJobs])) as Holder[]; } diff --git a/tests/unit/concurrency.test.ts b/tests/unit/concurrency.test.ts index d5698eb..1596a88 100644 --- a/tests/unit/concurrency.test.ts +++ b/tests/unit/concurrency.test.ts @@ -43,3 +43,70 @@ it('enforces per-host removal and recovery limits independently of other hosts', expect(concurrencyBlocker('b','restart','host')).toContain('recovery'); expect(concurrencyBlocker('b','restart','other')).toBeNull(); }); + +it('retains every capacity owner across failure, cancellation and migration without counting completed history', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { createHost } = await import('../../src/lib/server/hosts/index.js'); + const { lifecycleHolders, preparingJobs } = await import('../../src/lib/server/provisioning/concurrency.js'); + const h = db(), host = createHost({name:'Source'}); + const add = (id: string, state: string, command?: string, migration = false) => { + h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,?,'rebuild',?,'test','2026-01-01','test')").run(id,id,state); + h.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'lib','img',1,'2026-01-01')").run(id,id,host.id); + h.prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,?,1,?)').run(id,id,migration ? null : command ? id : null); + if (command) h.prepare("INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.rebuild',?,'{}','2026-01-01','2026-01-01')").run(id,host.id,command); + if (migration) h.prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id) VALUES(?,?,?,?,?)').run(id,host.id,id,'target',command ? id : null); + }; + add('running','running','running'); + add('health','awaiting-health','completed'); + add('prepared','queued'); + add('recovery','failed','failed'); + add('cancelled-command','cancelled','unknown'); + add('done-command','done','pending'); + add('failed-migration','failed','completed',true); + add('unknown-migration','legacy-unknown','completed',true); + add('cancelled-migration','cancelled','accepted',true); + add('waiting-migration','waiting','running',true); + add('finished','done','completed'); + add('cancelled-finished','cancelled','completed',true); + add('finished-migration','warning','completed',true); + add('queued','queued'); + h.prepare("INSERT INTO servers(name) VALUES('recovery')").run(); + h.prepare("INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('recovery','recovery','2026-01-01')").run(); + preparingJobs.add('prepared'); + try { + const holders = lifecycleHolders(); + expect(holders.map(row=>row.id).sort()).toEqual(['running','health','prepared','recovery','cancelled-command','done-command','failed-migration','unknown-migration','cancelled-migration','waiting-migration'].sort()); + expect(holders.find(row=>row.id==='waiting-migration')).toMatchObject({host_id:host.id,target_host_id:'target',removing:1}); + } finally { preparingJobs.clear(); } +}); + +it('seeks capacity owners through active indexes instead of scanning retained jobs and receipts', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { createHost } = await import('../../src/lib/server/hosts/index.js'); + const { lifecycleHolders } = await import('../../src/lib/server/provisioning/concurrency.js'); + const h = db(), host = createHost({name:'History host'}); + h.transaction(() => { + const job = h.prepare("INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'old','restart','done','Done','2026-01-01','test')"); + const operation = h.prepare("INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.restart','completed','{}','2026-01-01','2026-01-01')"); + h.prepare("INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','old',?,'lib','img',1,'2026-01-01')").run(host.id); + const native = h.prepare("INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,'i',1,?)"); + const migration = h.prepare("INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id,removed_at) VALUES(?,?,'i',?,'2026-01-01')"); + for (let n=0;n<4000;n++) { + const id=String(n); job.run(id); operation.run(id,host.id); native.run(id,id); migration.run(id,host.id,id); + } + })(); + const prepare = vi.spyOn(h,'prepare'); + let sql: string; + try { + expect(lifecycleHolders()).toEqual([]); + sql = prepare.mock.calls.find(([sql])=>sql.startsWith('SELECT d.id,d.action'))![0]; + } finally { prepare.mockRestore(); } + const plan = h.prepare(`EXPLAIN QUERY PLAN ${sql}`).all('[]') as {detail:string}[]; + const details = plan.map(row=>row.detail).join('\n'); + for (const index of ['idx_server_deployments_state','idx_server_deployments_unsettled','ec_host_operations_active','native_provisioning_operation','ec_host_migrations_operation']) { + expect(details).toContain(index); + } + expect(details).not.toMatch(/SCAN (?:d|n|m|o)\b/); + // Scanning the small partial index of unsettled IDs is intentional. + expect(details).not.toMatch(/SCAN server_deployments(?! USING INDEX idx_server_deployments_unsettled)\b/); +}); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index d2f84b8..205e53e 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -136,7 +136,8 @@ describe('provisioning reads with retained lifecycle history', () => { const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[]; expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([ 'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at', - 'idx_server_deployments_submission', 'idx_server_deployments_finished_at' + 'idx_server_deployments_submission', 'idx_server_deployments_finished_at', + 'idx_server_deployments_server_state', 'idx_server_deployments_unsettled' ])); expect(handle.pragma('foreign_key_check')).toEqual([]); closeDb();