import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; import type { HostInventory } from '../../src/lib/hosts.js'; import type { RebalanceSettings } from '../../src/lib/rebalancing.js'; // Artifact preservation has its own migration tests; these exercise live admission evidence. vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({ captureMoveRecipe: vi.fn(() => ({})) })); const server = 't6_zombie_town_1', gib = 1024 ** 3; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); }); async function setup() { const source = await prepareNativeHost([server]), target = await prepareNativeHost(); const { db } = await import('../../src/lib/server/db/index.js'); const { ensureRebalancingSchema } = await import('../../src/lib/server/rebalancing/schema.js'); ensureRebalancingSchema(db()); const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base); const at = new Date().toISOString(); const native = db().prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?').get(server) as { id: string; generation: number; library_id: string }; const original = db().prepare('SELECT inventory FROM ec_hosts WHERE id=?').get(source.host.id) as { inventory: string }; const inventory = JSON.parse(original.inventory) as HostInventory; const resources = { instanceId: native.id, generation: native.generation, cpuCores: 2, memoryBytes: 2 * gib, uptimeSeconds: 600 }; db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 75, memoryTotal: 16 * gib, memoryAvailable: 8 * gib, instanceResources: [resources] }), at, source.host.id); db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 0, memoryTotal: 16 * gib, memoryAvailable: 15 * gib, instanceResources: [] }), at, target.host.id); // Target has a verified copy of the source's deployed library version. const cache = db().prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?').get(native.library_id) as { manifest_sha256: string }; const libraries = await import('../../src/lib/server/game-files/index.js'); libraries.updateHostLibraryCaches(target.host.id, [{ libraryVersionId: native.library_id, status: 'ready', manifestSha256: cache.manifest_sha256 }]); db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(server, at, at, at); db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')").run(); db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key')").run(server); const settings: RebalanceSettings = { hostIds: [source.host.id, target.host.id], game: 'all', maxMoves: 10 }; const api = await import('../../src/lib/server/rebalancing/snapshot.js'); const planner = await import('../../src/lib/server/rebalancing/planner.js'); const read = (except = '') => api.rebalanceSnapshot(settings, except); return { db, source, target, native, inventory, resources, settings, read, ...api, ...planner, base, at }; } it('selects healthy empty native instances on active hosts without creating host work', async () => { const s = await setup(); const before = s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get(); const snapshot = s.read(); expect(snapshot.instances).toHaveLength(1); const plan = s.planRebalance(snapshot, s.settings); expect(plan.moves).toHaveLength(1); expect(plan.moves[0]).toMatchObject({ sourceHostId: s.source.host.id, targetHostId: s.target.host.id }); expect(s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get()).toEqual(before); expect(s.rebalanceHosts().every(host => host.eligible)).toBe(true); expect(s.read().hosts.find(host => host.id === s.source.host.id)?.used.cpuCores).toBe(6); }); it.each(['occupied', 'unknown', 'stopped', 'maintenance', 'recovery', 'lifecycle', 'generation', 'stale-host', 'future-host', 'startup', 'artifacts'] as const) ('excludes %s instances with an explanation', async condition => { const s = await setup(); if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run(); if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run(); if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run(); if (condition === 'maintenance') s.db().prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)').run(server, s.at); if (condition === 'recovery') s.db().prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, s.at); if (condition === 'lifecycle') s.db().prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at) VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`).run(s.source.host.id, s.native.id, s.at, s.at); if (condition === 'generation') s.db().prepare('UPDATE native_instances SET generation=2').run(); if (condition === 'stale-host' || condition === 'future-host') s.db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?') .run(new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(), s.source.host.id); if (condition === 'startup') s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?").run(s.source.host.id); if (condition === 'artifacts') vi.mocked((await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe).mockImplementationOnce(() => { throw new Error('Deployed files are unavailable.'); }); const snapshot = s.read(); expect(snapshot.instances).toEqual([]); expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) }); }); it('honors a current positive RCON veto even when telemetry reports empty', async () => { const s = await setup(), status = await import('../../src/lib/server/fleet/status.js'); const observe = status.observationReader(); vi.spyOn(status, 'observationReader').mockReturnValue(row => ({ ...observe(row), last_players: 0, status_source: 'telemetry' })); s.db().prepare('UPDATE servers SET last_players=1').run(); expect(s.read().exclusions[0].reason).toContain('RCON'); }); it('uses conservative recent host pressure and credits only matching observed generation relief', async () => { const s = await setup(); s.db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,8,90,?,?)`).run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib); s.db().prepare(`INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,1,'running',?,1,?)`).run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib); const snapshot = s.read(); expect(snapshot.hosts.find(host => host.id === s.source.host.id)?.used).toEqual({ cpuCores: 7.2, memoryBytes: 10 * gib }); expect(snapshot.instances[0].relief).toEqual({ cpuCores: 1, memoryBytes: gib }); expect(snapshot.instances[0].demand).toEqual({ cpuCores: 2, memoryBytes: 2 * gib }); }); it('counts a pending migration once before and after ownership changes and never assumes unplaced pinning', async () => { const s = await setup(); s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('moving',?,'rebuild','waiting','Move',?,'test')`).run(server, s.at); s.db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run('moving', s.native.id); s.db().prepare(`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id) VALUES('moving',?,?,?)`).run(s.source.host.id, s.native.id, s.target.host.id); const target = () => s.read().hosts.find(host => host.id === s.target.host.id)!; expect(target().used.cpuCores).toBe(0.5); expect(target().used.memoryBytes).toBe(2 * gib); expect(s.read('moving').hosts.find(host => host.id === s.target.host.id)?.used.cpuCores).toBe(0); s.db().prepare('UPDATE native_instances SET host_id=?,generation=2').run(s.target.host.id); s.db().prepare('UPDATE native_provisioning SET generation=2').run(); s.db().prepare('UPDATE ec_host_migrations SET removed_at=?').run(s.at); expect(target().used.cpuCores).toBe(0.5); expect(target().used.memoryBytes).toBe(2 * gib); s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?") .run(JSON.stringify([{ ...s.resources, generation: 2 }]), 13 * gib, s.target.host.id); expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib }); s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`).run(s.at); s.db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run('unplaced', JSON.stringify({ hostId: s.target.host.id })); expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib }); expect(target().instances).toBe(1); }); it('allows admission to ignore its own unsubmitted rebuild while retaining the source empty evidence', async () => { const s = await setup(); s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`).run(server, s.at); expect(s.read().instances).toHaveLength(0); expect(s.read('own').instances).toHaveLength(1); }); it('enforces the 30 minute cooldown across generations after a successful move', async () => { const s = await setup(); s.db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`).run(s.at, s.at); s.db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at) VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`).run(server, s.native.id, s.target.host.id, s.source.host.id, s.at); expect(s.read().exclusions[0].reason).toContain('30 minutes'); s.db().prepare('UPDATE rebalancing_moves SET updated_at=?').run(new Date(s.base - 30 * 60_000 - 1).toISOString()); expect(s.read().instances).toHaveLength(1); }); it('credits a freed game port only when the current range and bind will actually gain a port', async () => { const s = await setup(); expect(s.read().instances[0].releasesPort).toBe(false); s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,'0.0.0.0','udp',28960,?)`).run(s.source.host.id, s.native.id, s.at); expect(s.read().instances[0].releasesPort).toBe(true); s.db().prepare('UPDATE ec_hosts SET port_min=28970 WHERE id=?').run(s.source.host.id); expect(s.read().instances[0].releasesPort).toBe(false); s.db().prepare('UPDATE ec_hosts SET port_min=28960 WHERE id=?').run(s.source.host.id); s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,'other','127.0.0.2','tcp',28960,?)`).run(s.source.host.id, s.at); expect(s.read().instances[0].releasesPort).toBe(false); });