226 lines
11 KiB
TypeScript
226 lines
11 KiB
TypeScript
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture(['t5_test']);
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
it('counts rebuilds once across pools and keeps their capacity through migration handoff and lower limits', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { concurrencyBlocker, lifecycleHolders, preparingJobs } =
|
|
await import('../../src/lib/server/provisioning/concurrency.js');
|
|
const h = db();
|
|
h.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','source',1,'lib','img',1,'2026-01-01')"
|
|
).run();
|
|
h.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','rebuild','waiting','migrating','2026-01-01','test')"
|
|
).run();
|
|
h.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)"
|
|
).run();
|
|
h.prepare(
|
|
"INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id,removed_at) VALUES('a','source','i','target','remove','2026-01-02')"
|
|
).run();
|
|
saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 1, rebuildConcurrency: 1 });
|
|
expect(lifecycleHolders()).toHaveLength(1);
|
|
expect(concurrencyBlocker('b', 'stop', 'other')).toContain('lifecycle');
|
|
expect(concurrencyBlocker('a', 'rebuild', 'target')).toBeNull();
|
|
saveFleetSettings({ ...fleetSettings(), lifecycleConcurrency: 0 });
|
|
expect(concurrencyBlocker('b', 'rebuild', 'other')).toContain('rebuild');
|
|
saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 0, createConcurrencyPerHost: 1 });
|
|
expect(concurrencyBlocker('b', 'create', 'target')).toContain('startup');
|
|
expect(concurrencyBlocker('b', 'create', 'other')).toBeNull();
|
|
preparingJobs.add('a');
|
|
expect(lifecycleHolders()).toHaveLength(1);
|
|
preparingJobs.clear();
|
|
});
|
|
|
|
it('enforces per-host removal and recovery limits independently of other hosts', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { concurrencyBlocker } = await import('../../src/lib/server/provisioning/concurrency.js');
|
|
const h = db();
|
|
h.prepare("INSERT INTO servers(name) VALUES('t5_test')").run();
|
|
h.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('i','t5_test','host',1,'lib','img',1,'2026-01-01')"
|
|
).run();
|
|
h.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','destroy','running','removing','2026-01-01','test')"
|
|
).run();
|
|
h.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES('a','i',1)"
|
|
).run();
|
|
saveFleetSettings({
|
|
...fleetSettings(),
|
|
destroyConcurrencyPerHost: 1,
|
|
restartConcurrencyPerHost: 1
|
|
});
|
|
expect(concurrencyBlocker('b', 'destroy', 'host')).toContain('destruction');
|
|
expect(concurrencyBlocker('b', 'destroy', 'other')).toBeNull();
|
|
h.prepare(
|
|
"INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('t5_test','2026-01-01','a')"
|
|
).run();
|
|
expect(concurrencyBlocker('b', 'restart', 'host')).toContain('recovery');
|
|
expect(concurrencyBlocker('b', 'restart', 'other')).toBeNull();
|
|
});
|
|
|
|
it('counts a creation recovery rebuild against rebuild capacity without losing its startup reservation', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { concurrencyBlocker, lifecycleHolders } =
|
|
await import('../../src/lib/server/provisioning/concurrency.js');
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','t5_test','host','lib','img',1,'2026-01-01')"
|
|
)
|
|
.run();
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('a','t5_test','create','running','recovery','2026-01-01','test')"
|
|
)
|
|
.run();
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation,recovery_step) VALUES('a','i',1,'rebuild')"
|
|
)
|
|
.run();
|
|
saveFleetSettings({ ...fleetSettings(), rebuildConcurrency: 1, createConcurrencyPerHost: 1 });
|
|
expect(lifecycleHolders()).toEqual([expect.objectContaining({ id: 'a', action: 'rebuild' })]);
|
|
expect(concurrencyBlocker('b', 'rebuild', 'other')).toContain('rebuild');
|
|
expect(concurrencyBlocker('b', 'create', 'host')).toContain('startup');
|
|
expect(concurrencyBlocker('a', 'rebuild', 'host')).toBeNull();
|
|
});
|
|
|
|
it('retains every capacity owner across failure, cancellation and migration without counting completed history', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { lifecycleHolders, preparingJobs } =
|
|
await import('../../src/lib/server/provisioning/concurrency.js');
|
|
const h = db(),
|
|
host = createHost({ name: 'Source' });
|
|
const add = (id: string, state: string, command?: string, migration = false) => {
|
|
h.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,?,'rebuild',?,'test','2026-01-01','test')"
|
|
).run(id, id, state);
|
|
h.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'lib','img',1,'2026-01-01')"
|
|
).run(id, id, host.id);
|
|
h.prepare(
|
|
'INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,?,1,?)'
|
|
).run(id, id, migration ? null : command ? id : null);
|
|
if (command)
|
|
h.prepare(
|
|
"INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.rebuild',?,'{}','2026-01-01','2026-01-01')"
|
|
).run(id, host.id, command);
|
|
if (migration)
|
|
h.prepare(
|
|
'INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id,remove_operation_id) VALUES(?,?,?,?,?)'
|
|
).run(id, host.id, id, 'target', command ? id : null);
|
|
};
|
|
add('running', 'running', 'running');
|
|
add('health', 'awaiting-health', 'completed');
|
|
add('prepared', 'queued');
|
|
add('recovery', 'failed', 'failed');
|
|
add('cancelled-command', 'cancelled', 'unknown');
|
|
add('done-command', 'done', 'pending');
|
|
add('failed-migration', 'failed', 'completed', true);
|
|
add('unknown-migration', 'legacy-unknown', 'completed', true);
|
|
add('cancelled-migration', 'cancelled', 'accepted', true);
|
|
add('waiting-migration', 'waiting', 'running', true);
|
|
add('finished', 'done', 'completed');
|
|
add('cancelled-finished', 'cancelled', 'completed', true);
|
|
add('finished-migration', 'warning', 'completed', true);
|
|
add('queued', 'queued');
|
|
h.prepare("INSERT INTO servers(name) VALUES('recovery')").run();
|
|
h.prepare(
|
|
"INSERT INTO restart_rollout(server,operation_id,requested_at) VALUES('recovery','recovery','2026-01-01')"
|
|
).run();
|
|
preparingJobs.add('prepared');
|
|
try {
|
|
const holders = lifecycleHolders();
|
|
expect(holders.map((row) => row.id).sort()).toEqual(
|
|
[
|
|
'running',
|
|
'health',
|
|
'prepared',
|
|
'recovery',
|
|
'cancelled-command',
|
|
'done-command',
|
|
'failed-migration',
|
|
'unknown-migration',
|
|
'cancelled-migration',
|
|
'waiting-migration'
|
|
].sort()
|
|
);
|
|
expect(holders.find((row) => row.id === 'waiting-migration')).toMatchObject({
|
|
host_id: host.id,
|
|
target_host_id: 'target',
|
|
removing: 1
|
|
});
|
|
} finally {
|
|
preparingJobs.clear();
|
|
}
|
|
});
|
|
|
|
it('seeks capacity owners through active indexes instead of scanning retained jobs and receipts', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { lifecycleHolders } = await import('../../src/lib/server/provisioning/concurrency.js');
|
|
const h = db(),
|
|
host = createHost({ name: 'History host' });
|
|
h.transaction(() => {
|
|
const job = h.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES(?,'old','restart','done','Done','2026-01-01','test')"
|
|
);
|
|
const operation = h.prepare(
|
|
"INSERT INTO ec_host_operations(id,host_id,kind,state,payload,created_at,updated_at) VALUES(?,?,'instance.restart','completed','{}','2026-01-01','2026-01-01')"
|
|
);
|
|
h.prepare(
|
|
"INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('i','old',?,'lib','img',1,'2026-01-01')"
|
|
).run(host.id);
|
|
const native = h.prepare(
|
|
"INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id) VALUES(?,'i',1,?)"
|
|
);
|
|
const migration = h.prepare(
|
|
"INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,remove_operation_id,removed_at) VALUES(?,?,'i',?,'2026-01-01')"
|
|
);
|
|
for (let n = 0; n < 4000; n++) {
|
|
const id = String(n);
|
|
job.run(id);
|
|
operation.run(id, host.id);
|
|
native.run(id, id);
|
|
migration.run(id, host.id, id);
|
|
}
|
|
})();
|
|
const prepare = vi.spyOn(h, 'prepare');
|
|
let sql: string;
|
|
try {
|
|
expect(lifecycleHolders()).toEqual([]);
|
|
sql = prepare.mock.calls.find(([sql]) => sql.startsWith('SELECT d.id,'))![0];
|
|
} finally {
|
|
prepare.mockRestore();
|
|
}
|
|
const plan = h.prepare(`EXPLAIN QUERY PLAN ${sql}`).all('[]') as { detail: string }[];
|
|
const details = plan.map((row) => row.detail).join('\n');
|
|
for (const index of [
|
|
'idx_server_deployments_state',
|
|
'idx_server_deployments_unsettled',
|
|
'ec_host_operations_active',
|
|
'native_provisioning_operation',
|
|
'ec_host_migrations_operation'
|
|
]) {
|
|
expect(details).toContain(index);
|
|
}
|
|
expect(details).not.toMatch(/SCAN (?:d|n|m|o)\b/);
|
|
// Scanning the small partial index of unsettled IDs is intentional.
|
|
expect(details).not.toMatch(
|
|
/SCAN server_deployments(?! USING INDEX idx_server_deployments_unsettled)\b/
|
|
);
|
|
});
|