josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+126 -11
View File
@@ -543,23 +543,138 @@ describe('native instance provisioning', () => {
expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
});
it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
it('drains by reserving destination capacity, waiting for empty lobbies, and rebuilding once on another host', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { updateHost, createHost, createEnrollment, enrollHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
updateHost(hostId, { mode: 'draining' });
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/);
remote.players = 2; await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const retirement = db().prepare("SELECT id FROM server_deployments WHERE action='destroy' AND server=?").get(job.name) as { id: string };
expect(retirement).toBeTruthy(); expect(await operationFor(retirement.id)).toBeNull();
remote.players = 0; await advance();
const remove = await acknowledge(retirement.id);
expect(remove.kind).toBe('instance.remove');
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(remove.instanceId)).toEqual([]);
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
await processProvisioning();
const migration = db().prepare("SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?").get(job.name) as { id: string };
expect((await detail(migration.id)).stage).toContain('No host has capacity');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
const target = createHost({ name: 'Destination', portMin: 29000, portMax: 29005 });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
remote.players = 2; await processProvisioning();
expect((await detail(migration.id)).stage).toContain('2 player(s)');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
remote.players = 0; await processProvisioning();
const remove = db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").get() as { id: string };
expect(remove).toBeTruthy();
expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
expect(() => updateHost(target.id, { mode: 'draining' })).toThrow('incoming');
reportHostOperation(hostId, remove.id, { state: 'unknown' });
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
await processProvisioning();
expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
reportHostOperation(hostId, remove.id, { state: 'completed' });
await processProvisioning();
expect(db().prepare('SELECT host_id,generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: target.id, generation: 2 });
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
expect(getHost(hostId)?.mode).toBe('draining');
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
expect(cancelProvisioning(migration.id)).toBe(false);
await processProvisioning();
const create = (await operationFor(migration.id))!;
expect(create).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2, payload: { name: job.name, port: 29000 } });
reportHostOperation(target.id, create.id, { state: 'completed', result: { state: 'running', healthy: true } });
await processProvisioning();
expect((await detail(migration.id)).state).toBe('done');
expect(getHost(hostId)?.mode).toBe('disabled');
expect(db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ generation: 2 });
await advance();
expect((await operationFor(migration.id))?.id).toBe(create.id);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(1);
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
});
it('reserves migration capacity before source removal and retains it while a replacement awaits startup', async () => {
const first = await queue(); await advance(); await acknowledge(first.deploymentId);
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
if (!second.ok) throw new Error(second.message);
await advance(); await acknowledge(second.deploymentId);
const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'One free slot' });
enrollHost(createEnrollment(target.id).token, { ...inventory(), memoryTotal: 4 * 1024 ** 3, memoryAvailable: 1.75 * 1024 ** 3 });
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const removals = () => db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() as { id: string }[];
expect(removals()).toHaveLength(1);
expect(db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
reportHostOperation(hostId, removals()[0].id, { state: 'completed' });
await advance();
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
expect(removals()).toHaveLength(1);
});
it('retries definite migration failures without replaying successful source removal', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'Destination' });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as { deployment_id: string; remove_operation_id: string };
const expireRetry = (id: string) => db().prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(new Date(Date.now() - 61_000).toISOString(), id);
reportHostOperation(hostId, migration.remove_operation_id, { state: 'failed', error: 'Docker unavailable before removal' });
await advance();
expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
expireRetry(migration.remove_operation_id);
await advance(); await advance();
const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as { remove_operation_id: string };
expect(retry.remove_operation_id).not.toBe(migration.remove_operation_id);
reportHostOperation(hostId, retry.remove_operation_id, { state: 'completed' });
await advance(); await advance();
const creation = (await operationFor(migration.deployment_id))!;
reportHostOperation(target.id, creation.id, { state: 'failed', error: 'Docker unavailable before creation' });
expireRetry(creation.id);
await advance(); await advance();
const replacement = (await operationFor(migration.deployment_id))!;
expect(replacement.id).not.toBe(creation.id);
expect(replacement).toMatchObject({ kind: 'instance.create', hostId: target.id, generation: 2 });
reportHostOperation(target.id, replacement.id, { state: 'completed' });
await advance();
expect(getHost(hostId)?.mode).toBe('disabled');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(2);
});
it('disables an empty draining host and redirects its unplaced launches to another host', async () => {
const job = await queue();
const { createHost, createEnrollment, enrollHost, updateHost, getHost } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'Destination' });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
expect(() => updateHost(hostId, { mode: 'active' })).toThrow('finish draining');
await advance();
expect(getHost(hostId)?.mode).toBe('disabled');
expect(await operationFor(job.deploymentId)).toMatchObject({ hostId: target.id, kind: 'instance.create' });
});
it('rejects the removed manual disable API action', async () => {
const { PATCH } = await import('../../src/routes/api/hosts/[id]/+server.js');
const response = await PATCH({ params: { id: hostId }, request: new Request(`https://ec.test/api/hosts/${hostId}`, {
method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: 'disabled' })
}) });
expect(response.status).toBe(400);
expect(await response.json()).toMatchObject({ error: expect.stringContaining('disable automatically') });
const { getHost } = await import('../../src/lib/server/hosts/index.js');
expect(getHost(hostId)?.mode).toBe('active');
});
it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { db } = await import('../../src/lib/server/db/index.js');