josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/rebalancing-snapshot.test.ts
+250 -70
View File
@@ -5,40 +5,118 @@ import type { HostInventory } from '../../src/lib/hosts.js';
import type { RebalanceSettings } from '../../src/lib/rebalancing.js';
// Artifact preservation has its own migration tests; these exercise live admission evidence.
vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({ captureMoveRecipe: vi.fn(() => ({})) }));
const server = 't6_zombie_town_1', gib = 1024 ** 3;
vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({
captureMoveRecipe: vi.fn(() => ({}))
}));
const server = 't6_zombie_town_1',
gib = 1024 ** 3;
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.useRealTimers(); vi.restoreAllMocks(); await fixture.cleanup(); });
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
});
afterEach(async () => {
vi.useRealTimers();
vi.restoreAllMocks();
await fixture.cleanup();
});
async function setup() {
const source = await prepareNativeHost([server]), target = await prepareNativeHost();
const source = await prepareNativeHost([server]),
target = await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const { ensureRebalancingSchema } = await import('../../src/lib/server/rebalancing/schema.js');
ensureRebalancingSchema(db());
const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base);
const base = Date.now();
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(base);
const at = new Date().toISOString();
const native = db().prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?').get(server) as { id: string; generation: number; library_id: string };
const original = db().prepare('SELECT inventory FROM ec_hosts WHERE id=?').get(source.host.id) as { inventory: string };
const native = db()
.prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?')
.get(server) as { id: string; generation: number; library_id: string };
const original = db()
.prepare('SELECT inventory FROM ec_hosts WHERE id=?')
.get(source.host.id) as { inventory: string };
const inventory = JSON.parse(original.inventory) as HostInventory;
const resources = { instanceId: native.id, generation: native.generation, cpuCores: 2, memoryBytes: 2 * gib, uptimeSeconds: 600 };
db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 75,
memoryTotal: 16 * gib, memoryAvailable: 8 * gib, instanceResources: [resources] }), at, source.host.id);
db().prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?').run(JSON.stringify({ ...inventory, cpuCount: 8, cpuUsagePercent: 0,
memoryTotal: 16 * gib, memoryAvailable: 15 * gib, instanceResources: [] }), at, target.host.id);
const resources = {
instanceId: native.id,
generation: native.generation,
cpuCores: 2,
memoryBytes: 2 * gib,
uptimeSeconds: 600
};
db()
.prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?')
.run(
JSON.stringify({
...inventory,
cpuCount: 8,
cpuUsagePercent: 75,
memoryTotal: 16 * gib,
memoryAvailable: 8 * gib,
instanceResources: [resources]
}),
at,
source.host.id
);
db()
.prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?')
.run(
JSON.stringify({
...inventory,
cpuCount: 8,
cpuUsagePercent: 0,
memoryTotal: 16 * gib,
memoryAvailable: 15 * gib,
instanceResources: []
}),
at,
target.host.id
);
// Target has a verified copy of the source's deployed library version.
const cache = db().prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?').get(native.library_id) as { manifest_sha256: string };
const cache = db()
.prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?')
.get(native.library_id) as { manifest_sha256: string };
const libraries = await import('../../src/lib/server/game-files/index.js');
libraries.updateHostLibraryCaches(target.host.id, [{ libraryVersionId: native.library_id, status: 'ready', manifestSha256: cache.manifest_sha256 }]);
db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(server, at, at, at);
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')").run();
libraries.updateHostLibraryCaches(target.host.id, [
{
libraryVersionId: native.library_id,
status: 'ready',
manifestSha256: cache.manifest_sha256
}
]);
db()
.prepare(
`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`
)
.run(server, at, at, at);
db()
.prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')")
.run();
db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key')").run(server);
const settings: RebalanceSettings = { hostIds: [source.host.id, target.host.id], game: 'all', maxMoves: 10 };
const settings: RebalanceSettings = {
hostIds: [source.host.id, target.host.id],
game: 'all',
maxMoves: 10
};
const api = await import('../../src/lib/server/rebalancing/snapshot.js');
const planner = await import('../../src/lib/server/rebalancing/planner.js');
const read = (except = '') => api.rebalanceSnapshot(settings, except);
return { db, source, target, native, inventory, resources, settings, read, ...api, ...planner, base, at };
return {
db,
source,
target,
native,
inventory,
resources,
settings,
read,
...api,
...planner,
base,
at
};
}
it('selects healthy empty native instances on active hosts without creating host work', async () => {
@@ -48,106 +126,208 @@ it('selects healthy empty native instances on active hosts without creating host
expect(snapshot.instances).toHaveLength(1);
const plan = s.planRebalance(snapshot, s.settings);
expect(plan.moves).toHaveLength(1);
expect(plan.moves[0]).toMatchObject({ sourceHostId: s.source.host.id, targetHostId: s.target.host.id });
expect(plan.moves[0]).toMatchObject({
sourceHostId: s.source.host.id,
targetHostId: s.target.host.id
});
expect(s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get()).toEqual(before);
expect(s.rebalanceHosts().every(host => host.eligible)).toBe(true);
expect(s.read().hosts.find(host => host.id === s.source.host.id)?.used.cpuCores).toBe(6);
expect(s.rebalanceHosts().every((host) => host.eligible)).toBe(true);
expect(s.read().hosts.find((host) => host.id === s.source.host.id)?.used.cpuCores).toBe(6);
});
it.each(['occupied', 'unknown', 'stopped', 'maintenance', 'recovery', 'lifecycle', 'generation', 'stale-host', 'future-host', 'startup', 'artifacts'] as const)
('excludes %s instances with an explanation', async condition => {
const s = await setup();
if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run();
if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run();
if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run();
if (condition === 'maintenance') s.db().prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)').run(server, s.at);
if (condition === 'recovery') s.db().prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)').run(server, s.at);
if (condition === 'lifecycle') s.db().prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at)
VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`).run(s.source.host.id, s.native.id, s.at, s.at);
if (condition === 'generation') s.db().prepare('UPDATE native_instances SET generation=2').run();
if (condition === 'stale-host' || condition === 'future-host') s.db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
.run(new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(), s.source.host.id);
if (condition === 'startup') s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?").run(s.source.host.id);
if (condition === 'artifacts') vi.mocked((await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe).mockImplementationOnce(() => { throw new Error('Deployed files are unavailable.'); });
const snapshot = s.read();
expect(snapshot.instances).toEqual([]);
expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) });
});
it.each([
'occupied',
'unknown',
'stopped',
'maintenance',
'recovery',
'lifecycle',
'generation',
'stale-host',
'future-host',
'startup',
'artifacts'
] as const)('excludes %s instances with an explanation', async (condition) => {
const s = await setup();
if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run();
if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run();
if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run();
if (condition === 'maintenance')
s.db()
.prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)')
.run(server, s.at);
if (condition === 'recovery')
s.db()
.prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)')
.run(server, s.at);
if (condition === 'lifecycle')
s.db()
.prepare(
`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at)
VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`
)
.run(s.source.host.id, s.native.id, s.at, s.at);
if (condition === 'generation')
s.db().prepare('UPDATE native_instances SET generation=2').run();
if (condition === 'stale-host' || condition === 'future-host')
s.db()
.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
.run(
new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(),
s.source.host.id
);
if (condition === 'startup')
s.db()
.prepare(
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?"
)
.run(s.source.host.id);
if (condition === 'artifacts')
vi.mocked(
(await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe
).mockImplementationOnce(() => {
throw new Error('Deployed files are unavailable.');
});
const snapshot = s.read();
expect(snapshot.instances).toEqual([]);
expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) });
});
it('honors a current positive RCON veto even when telemetry reports empty', async () => {
const s = await setup(), status = await import('../../src/lib/server/fleet/status.js');
const s = await setup(),
status = await import('../../src/lib/server/fleet/status.js');
const observe = status.observationReader();
vi.spyOn(status, 'observationReader').mockReturnValue(row => ({ ...observe(row), last_players: 0, status_source: 'telemetry' }));
vi.spyOn(status, 'observationReader').mockReturnValue((row) => ({
...observe(row),
last_players: 0,
status_source: 'telemetry'
}));
s.db().prepare('UPDATE servers SET last_players=1').run();
expect(s.read().exclusions[0].reason).toContain('RCON');
});
it('uses conservative recent host pressure and credits only matching observed generation relief', async () => {
const s = await setup();
s.db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
VALUES(?,?,8,90,?,?)`).run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib);
s.db().prepare(`INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes)
VALUES(?,?,1,'running',?,1,?)`).run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib);
s.db()
.prepare(
`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
VALUES(?,?,8,90,?,?)`
)
.run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib);
s.db()
.prepare(
`INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes)
VALUES(?,?,1,'running',?,1,?)`
)
.run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib);
const snapshot = s.read();
expect(snapshot.hosts.find(host => host.id === s.source.host.id)?.used).toEqual({ cpuCores: 7.2, memoryBytes: 10 * gib });
expect(snapshot.hosts.find((host) => host.id === s.source.host.id)?.used).toEqual({
cpuCores: 7.2,
memoryBytes: 10 * gib
});
expect(snapshot.instances[0].relief).toEqual({ cpuCores: 1, memoryBytes: gib });
expect(snapshot.instances[0].demand).toEqual({ cpuCores: 2, memoryBytes: 2 * gib });
});
it('counts a pending migration once before and after ownership changes and never assumes unplaced pinning', async () => {
const s = await setup();
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('moving',?,'rebuild','waiting','Move',?,'test')`).run(server, s.at);
s.db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run('moving', s.native.id);
s.db().prepare(`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id)
VALUES('moving',?,?,?)`).run(s.source.host.id, s.native.id, s.target.host.id);
const target = () => s.read().hosts.find(host => host.id === s.target.host.id)!;
s.db()
.prepare(
`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('moving',?,'rebuild','waiting','Move',?,'test')`
)
.run(server, s.at);
s.db()
.prepare(
'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)'
)
.run('moving', s.native.id);
s.db()
.prepare(
`INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id)
VALUES('moving',?,?,?)`
)
.run(s.source.host.id, s.native.id, s.target.host.id);
const target = () => s.read().hosts.find((host) => host.id === s.target.host.id)!;
expect(target().used.cpuCores).toBe(0.5);
expect(target().used.memoryBytes).toBe(2 * gib);
expect(s.read('moving').hosts.find(host => host.id === s.target.host.id)?.used.cpuCores).toBe(0);
expect(s.read('moving').hosts.find((host) => host.id === s.target.host.id)?.used.cpuCores).toBe(
0
);
s.db().prepare('UPDATE native_instances SET host_id=?,generation=2').run(s.target.host.id);
s.db().prepare('UPDATE native_provisioning SET generation=2').run();
s.db().prepare('UPDATE ec_host_migrations SET removed_at=?').run(s.at);
expect(target().used.cpuCores).toBe(0.5);
expect(target().used.memoryBytes).toBe(2 * gib);
s.db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?")
s.db()
.prepare(
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?"
)
.run(JSON.stringify([{ ...s.resources, generation: 2 }]), 13 * gib, s.target.host.id);
expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`).run(s.at);
s.db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run('unplaced', JSON.stringify({ hostId: s.target.host.id }));
s.db()
.prepare(
`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`
)
.run(s.at);
s.db()
.prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)')
.run('unplaced', JSON.stringify({ hostId: s.target.host.id }));
expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
expect(target().instances).toBe(1);
});
it('allows admission to ignore its own unsubmitted rebuild while retaining the source empty evidence', async () => {
const s = await setup();
s.db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`).run(server, s.at);
s.db()
.prepare(
`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`
)
.run(server, s.at);
expect(s.read().instances).toHaveLength(0);
expect(s.read('own').instances).toHaveLength(1);
});
it('enforces the 30 minute cooldown across generations after a successful move', async () => {
const s = await setup();
s.db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`).run(s.at, s.at);
s.db().prepare(`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at)
VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`).run(server, s.native.id, s.target.host.id, s.source.host.id, s.at);
s.db()
.prepare(
`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`
)
.run(s.at, s.at);
s.db()
.prepare(
`INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at)
VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`
)
.run(server, s.native.id, s.target.host.id, s.source.host.id, s.at);
expect(s.read().exclusions[0].reason).toContain('30 minutes');
s.db().prepare('UPDATE rebalancing_moves SET updated_at=?').run(new Date(s.base - 30 * 60_000 - 1).toISOString());
s.db()
.prepare('UPDATE rebalancing_moves SET updated_at=?')
.run(new Date(s.base - 30 * 60_000 - 1).toISOString());
expect(s.read().instances).toHaveLength(1);
});
it('credits a freed game port only when the current range and bind will actually gain a port', async () => {
const s = await setup();
expect(s.read().instances[0].releasesPort).toBe(false);
s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
VALUES(?,?,'0.0.0.0','udp',28960,?)`).run(s.source.host.id, s.native.id, s.at);
s.db()
.prepare(
`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
VALUES(?,?,'0.0.0.0','udp',28960,?)`
)
.run(s.source.host.id, s.native.id, s.at);
expect(s.read().instances[0].releasesPort).toBe(true);
s.db().prepare('UPDATE ec_hosts SET port_min=28970 WHERE id=?').run(s.source.host.id);
expect(s.read().instances[0].releasesPort).toBe(false);
s.db().prepare('UPDATE ec_hosts SET port_min=28960 WHERE id=?').run(s.source.host.id);
s.db().prepare(`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
VALUES(?,'other','127.0.0.2','tcp',28960,?)`).run(s.source.host.id, s.at);
s.db()
.prepare(
`INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
VALUES(?,'other','127.0.0.2','tcp',28960,?)`
)
.run(s.source.host.id, s.at);
expect(s.read().instances[0].releasesPort).toBe(false);
});