josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/host-preparation.test.ts
+77
View File
@@ -68,6 +68,83 @@ function verified(id: string, version: GameLibraryVersion) {
}
describe('automatic host preparation', () => {
it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => {
const t5 = await publish('t5');
const t6 = await publish('t6');
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
const id = enroll(readyInventory);
verified(id, t5); verified(id, t6);
const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' });
heartbeat(id, readyInventory);
hosts.updateHost(id, { mode: 'disabled' });
const requested = hosts.updateHost(id, { mode: 'active' });
expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) });
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
preparation.reconcileHostPreparation(id);
const pull = operations(id)[0];
expect(pull.id).not.toBe(oldRuntime.id);
expect(pull.kind).toBe('runtime.pull');
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]);
expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled');
expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince);
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
preparation.reconcileHostPreparation(id);
expect(hosts.getHost(id)?.mode).toBe('disabled');
heartbeat(id, readyInventory);
for (let index = 0; index < 2; index++) {
preparation.reconcileHostPreparation(id);
const download = operations(id)[0];
expect(download.kind).toBe('library.download');
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]);
const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!;
verified(id, version);
hosts.reportHostOperation(id, download.id, { state: 'completed' });
preparation.reconcileHostPreparation(id);
expect(hosts.getHost(id)?.mode).toBe('disabled');
heartbeat(id, readyInventory);
}
preparation.reconcileHostPreparation(id);
expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null });
expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2);
});
it('requires both games when enabling even if one library is already cached', async () => {
const version = await publish('t5');
const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
verified(id, version);
hosts.updateHost(id, { mode: 'disabled' });
hosts.updateHost(id, { mode: 'active' });
const status = preparation.reconcileHostPreparation(id);
expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II (T6)', state: 'blocked' }));
expect(hosts.getHost(id)?.mode).toBe('disabled');
});
it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => {
const t5 = await publish('t5');
const t6 = await publish('t6');
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
const id = enroll(readyInventory);
verified(id, t5); verified(id, t6);
hosts.updateHost(id, { mode: 'disabled' });
hosts.updateHost(id, { mode: 'active' });
preparation.reconcileHostPreparation(id);
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' });
heartbeat(id, readyInventory);
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
const queued = operations(id).map(operation => operation.id);
vi.setSystemTime(Date.now() + 91_000);
preparation.reconcileHostPreparation(id);
expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false });
expect(operations(id).map(operation => operation.id)).toEqual(queued);
heartbeat(id, readyInventory);
preparation.reconcileHostPreparation(id);
expect(hosts.getHost(id)?.mode).toBe('disabled');
});
it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([
import('../../src/routes/api/agent/enroll/+server.js'),
tests/unit/native-launch-preparation.test.ts
+3 -3
View File
@@ -218,7 +218,7 @@ describe('asynchronous native launch preparation', () => {
expect(await operationFor(second.deploymentId)).toBeUndefined();
});
it('keeps a prepared launch queued if its host is disabled before dispatch', async () => {
it('keeps a prepared launch queued while its host is disabled or re-enabling', async () => {
const job = await queue();
const paused = await pausePreparation();
const pass = runPass();
@@ -228,9 +228,9 @@ describe('asynchronous native launch preparation', () => {
paused.release();
await pass;
expect(await operationFor(job.deploymentId)).toBeUndefined();
updateHost(hostId, { mode: 'active' });
expect(updateHost(hostId, { mode: 'active' })).toMatchObject({ mode: 'disabled', readiness: 'preparing' });
await runPass();
expect(await operationFor(job.deploymentId)).toBeDefined();
expect(await operationFor(job.deploymentId)).toBeUndefined();
});
it('rechecks a reduced startup limit before submitting prepared files', async () => {
tests/unit/native-provisioning.test.ts
+126 -11
View File
@@ -543,23 +543,138 @@ describe('native instance provisioning', () => {
expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
});
it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
it('drains by reserving destination capacity, waiting for empty lobbies, and rebuilding once on another host', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { updateHost, createHost, createEnrollment, enrollHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
updateHost(hostId, { mode: 'draining' });
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/);
remote.players = 2; await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const retirement = db().prepare("SELECT id FROM server_deployments WHERE action='destroy' AND server=?").get(job.name) as { id: string };
expect(retirement).toBeTruthy(); expect(await operationFor(retirement.id)).toBeNull();
remote.players = 0; await advance();
const remove = await acknowledge(retirement.id);
expect(remove.kind).toBe('instance.remove');
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(remove.instanceId)).toEqual([]);
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
await processProvisioning();
const migration = db().prepare("SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?").get(job.name) as { id: string };
expect((await detail(migration.id)).stage).toContain('No host has capacity');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
const target = createHost({ name: 'Destination', portMin: 29000, portMax: 29005 });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
remote.players = 2; await processProvisioning();
expect((await detail(migration.id)).stage).toContain('2 player(s)');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
remote.players = 0; await processProvisioning();
const remove = db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").get() as { id: string };
expect(remove).toBeTruthy();
expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
expect(() => updateHost(target.id, { mode: 'draining' })).toThrow('incoming');
reportHostOperation(hostId, remove.id, { state: 'unknown' });
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
await processProvisioning();
expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
reportHostOperation(hostId, remove.id, { state: 'completed' });
await processProvisioning();
expect(db().prepare('SELECT host_id,generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: target.id, generation: 2 });
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
expect(getHost(hostId)?.mode).toBe('draining');
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
expect(cancelProvisioning(migration.id)).toBe(false);
await processProvisioning();
const create = (await operationFor(migration.id))!;
expect(create).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2, payload: { name: job.name, port: 29000 } });
reportHostOperation(target.id, create.id, { state: 'completed', result: { state: 'running', healthy: true } });
await processProvisioning();
expect((await detail(migration.id)).state).toBe('done');
expect(getHost(hostId)?.mode).toBe('disabled');
expect(db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ generation: 2 });
await advance();
expect((await operationFor(migration.id))?.id).toBe(create.id);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(1);
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
});
it('reserves migration capacity before source removal and retains it while a replacement awaits startup', async () => {
const first = await queue(); await advance(); await acknowledge(first.deploymentId);
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
if (!second.ok) throw new Error(second.message);
await advance(); await acknowledge(second.deploymentId);
const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'One free slot' });
enrollHost(createEnrollment(target.id).token, { ...inventory(), memoryTotal: 4 * 1024 ** 3, memoryAvailable: 1.75 * 1024 ** 3 });
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const removals = () => db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() as { id: string }[];
expect(removals()).toHaveLength(1);
expect(db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
reportHostOperation(hostId, removals()[0].id, { state: 'completed' });
await advance();
expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
expect(removals()).toHaveLength(1);
});
it('retries definite migration failures without replaying successful source removal', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'Destination' });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as { deployment_id: string; remove_operation_id: string };
const expireRetry = (id: string) => db().prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(new Date(Date.now() - 61_000).toISOString(), id);
reportHostOperation(hostId, migration.remove_operation_id, { state: 'failed', error: 'Docker unavailable before removal' });
await advance();
expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
expireRetry(migration.remove_operation_id);
await advance(); await advance();
const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as { remove_operation_id: string };
expect(retry.remove_operation_id).not.toBe(migration.remove_operation_id);
reportHostOperation(hostId, retry.remove_operation_id, { state: 'completed' });
await advance(); await advance();
const creation = (await operationFor(migration.deployment_id))!;
reportHostOperation(target.id, creation.id, { state: 'failed', error: 'Docker unavailable before creation' });
expireRetry(creation.id);
await advance(); await advance();
const replacement = (await operationFor(migration.deployment_id))!;
expect(replacement.id).not.toBe(creation.id);
expect(replacement).toMatchObject({ kind: 'instance.create', hostId: target.id, generation: 2 });
reportHostOperation(target.id, replacement.id, { state: 'completed' });
await advance();
expect(getHost(hostId)?.mode).toBe('disabled');
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(2);
});
it('disables an empty draining host and redirects its unplaced launches to another host', async () => {
const job = await queue();
const { createHost, createEnrollment, enrollHost, updateHost, getHost } = await import('../../src/lib/server/hosts/index.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const target = createHost({ name: 'Destination' });
enrollHost(createEnrollment(target.id).token, inventory());
updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
updateHost(hostId, { mode: 'draining' });
expect(() => updateHost(hostId, { mode: 'active' })).toThrow('finish draining');
await advance();
expect(getHost(hostId)?.mode).toBe('disabled');
expect(await operationFor(job.deploymentId)).toMatchObject({ hostId: target.id, kind: 'instance.create' });
});
it('rejects the removed manual disable API action', async () => {
const { PATCH } = await import('../../src/routes/api/hosts/[id]/+server.js');
const response = await PATCH({ params: { id: hostId }, request: new Request(`https://ec.test/api/hosts/${hostId}`, {
method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: 'disabled' })
}) });
expect(response.status).toBe(400);
expect(await response.json()).toMatchObject({ error: expect.stringContaining('disable automatically') });
const { getHost } = await import('../../src/lib/server/hosts/index.js');
expect(getHost(hostId)?.mode).toBe('active');
});
it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { db } = await import('../../src/lib/server/db/index.js');