electric-cherry/tests/unit/rebalancing-snapshot.test.ts

import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { prepareNativeHost } from '../helpers/native-host.js';
import type { HostInventory } from '../../src/lib/hosts.js';
import type { RebalanceSettings } from '../../src/lib/rebalancing.js';

// Artifact preservation has its own migration tests; these exercise live admission evidence.
vi.mock('../../src/lib/server/provisioning/move-recipes.js', () => ({
    captureMoveRecipe: vi.fn(() => ({}))
}));
const server = 't6_zombie_town_1',
    gib = 1024 ** 3;
let fixture: Fixture;
beforeEach(async () => {
    vi.resetModules();
    fixture = await newFixture([]);
});
afterEach(async () => {
    vi.useRealTimers();
    vi.restoreAllMocks();
    await fixture.cleanup();
});

async function setup() {
    const source = await prepareNativeHost([server]),
        target = await prepareNativeHost();
    const { db } = await import('../../src/lib/server/db/index.js');
    const { ensureRebalancingSchema } = await import('../../src/lib/server/rebalancing/schema.js');
    ensureRebalancingSchema(db());
    const base = Date.now();
    vi.useFakeTimers({ toFake: ['Date'] });
    vi.setSystemTime(base);
    const at = new Date().toISOString();
    const native = db()
        .prepare('SELECT id,generation,library_id FROM native_instances WHERE server=?')
        .get(server) as { id: string; generation: number; library_id: string };
    const original = db()
        .prepare('SELECT inventory FROM ec_hosts WHERE id=?')
        .get(source.host.id) as { inventory: string };
    const inventory = JSON.parse(original.inventory) as HostInventory;
    const resources = {
        instanceId: native.id,
        generation: native.generation,
        cpuCores: 2,
        memoryBytes: 2 * gib,
        uptimeSeconds: 600
    };
    db()
        .prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?')
        .run(
            JSON.stringify({
                ...inventory,
                cpuCount: 8,
                cpuUsagePercent: 75,
                memoryTotal: 16 * gib,
                memoryAvailable: 8 * gib,
                instanceResources: [resources]
            }),
            at,
            source.host.id
        );
    db()
        .prepare('UPDATE ec_hosts SET inventory=?,last_heartbeat=? WHERE id=?')
        .run(
            JSON.stringify({
                ...inventory,
                cpuCount: 8,
                cpuUsagePercent: 0,
                memoryTotal: 16 * gib,
                memoryAvailable: 15 * gib,
                instanceResources: []
            }),
            at,
            target.host.id
        );
    // Target has a verified copy of the source's deployed library version.
    const cache = db()
        .prepare('SELECT manifest_sha256 FROM game_file_versions WHERE id=?')
        .get(native.library_id) as { manifest_sha256: string };
    const libraries = await import('../../src/lib/server/game-files/index.js');
    libraries.updateHostLibraryCaches(target.host.id, [
        {
            libraryVersionId: native.library_id,
            status: 'ready',
            manifestSha256: cache.manifest_sha256
        }
    ]);
    db()
        .prepare(
            `INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
        VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`
        )
        .run(server, at, at, at);
    db()
        .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key','t6','private-key')")
        .run();
    db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key')").run(server);
    const settings: RebalanceSettings = {
        hostIds: [source.host.id, target.host.id],
        game: 'all',
        maxMoves: 10
    };
    const api = await import('../../src/lib/server/rebalancing/snapshot.js');
    const planner = await import('../../src/lib/server/rebalancing/planner.js');
    const read = (except = '') => api.rebalanceSnapshot(settings, except);
    return {
        db,
        source,
        target,
        native,
        inventory,
        resources,
        settings,
        read,
        ...api,
        ...planner,
        base,
        at
    };
}

it('selects healthy empty native instances on active hosts without creating host work', async () => {
    const s = await setup();
    const before = s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get();
    const snapshot = s.read();
    expect(snapshot.instances).toHaveLength(1);
    const plan = s.planRebalance(snapshot, s.settings);
    expect(plan.moves).toHaveLength(1);
    expect(plan.moves[0]).toMatchObject({
        sourceHostId: s.source.host.id,
        targetHostId: s.target.host.id
    });
    expect(s.db().prepare('SELECT COUNT(*) AS n FROM ec_host_operations').get()).toEqual(before);
    expect(s.rebalanceHosts().every((host) => host.eligible)).toBe(true);
    expect(s.read().hosts.find((host) => host.id === s.source.host.id)?.used.cpuCores).toBe(6);
});

it.each([
    'occupied',
    'unknown',
    'stopped',
    'maintenance',
    'recovery',
    'lifecycle',
    'generation',
    'stale-host',
    'future-host',
    'startup',
    'artifacts'
] as const)('excludes %s instances with an explanation', async (condition) => {
    const s = await setup();
    if (condition === 'occupied') s.db().prepare('UPDATE servers SET last_players=1').run();
    if (condition === 'unknown') s.db().prepare('UPDATE servers SET last_players=NULL').run();
    if (condition === 'stopped') s.db().prepare("UPDATE servers SET desired_state='stopped'").run();
    if (condition === 'maintenance')
        s.db()
            .prepare('INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)')
            .run(server, s.at);
    if (condition === 'recovery')
        s.db()
            .prepare('INSERT INTO restart_rollout(server,requested_at) VALUES(?,?)')
            .run(server, s.at);
    if (condition === 'lifecycle')
        s.db()
            .prepare(
                `INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,payload,state,created_at,updated_at)
            VALUES('pending',?,?,1,'instance.stop','{}','unknown',?,?)`
            )
            .run(s.source.host.id, s.native.id, s.at, s.at);
    if (condition === 'generation')
        s.db().prepare('UPDATE native_instances SET generation=2').run();
    if (condition === 'stale-host' || condition === 'future-host')
        s.db()
            .prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
            .run(
                new Date(s.base + (condition === 'stale-host' ? -91_000 : 1)).toISOString(),
                s.source.host.id
            );
    if (condition === 'startup')
        s.db()
            .prepare(
                "UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources[0].uptimeSeconds',200) WHERE id=?"
            )
            .run(s.source.host.id);
    if (condition === 'artifacts')
        vi.mocked(
            (await import('../../src/lib/server/provisioning/move-recipes.js')).captureMoveRecipe
        ).mockImplementationOnce(() => {
            throw new Error('Deployed files are unavailable.');
        });
    const snapshot = s.read();
    expect(snapshot.instances).toEqual([]);
    expect(snapshot.exclusions[0]).toMatchObject({ server, reason: expect.any(String) });
});

it('honors a current positive RCON veto even when telemetry reports empty', async () => {
    const s = await setup(),
        status = await import('../../src/lib/server/fleet/status.js');
    const observe = status.observationReader();
    vi.spyOn(status, 'observationReader').mockReturnValue((row) => ({
        ...observe(row),
        last_players: 0,
        status_source: 'telemetry'
    }));
    s.db().prepare('UPDATE servers SET last_players=1').run();
    expect(s.read().exclusions[0].reason).toContain('RCON');
});

it('uses conservative recent host pressure and credits only matching observed generation relief', async () => {
    const s = await setup();
    s.db()
        .prepare(
            `INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
        VALUES(?,?,8,90,?,?)`
        )
        .run(s.source.host.id, new Date(s.base - 60_000).toISOString(), 16 * gib, 6 * gib);
    s.db()
        .prepare(
            `INSERT INTO ec_instance_resource_samples(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes)
        VALUES(?,?,1,'running',?,1,?)`
        )
        .run(s.source.host.id, s.native.id, new Date(s.base - 60_000).toISOString(), gib);
    const snapshot = s.read();
    expect(snapshot.hosts.find((host) => host.id === s.source.host.id)?.used).toEqual({
        cpuCores: 7.2,
        memoryBytes: 10 * gib
    });
    expect(snapshot.instances[0].relief).toEqual({ cpuCores: 1, memoryBytes: gib });
    expect(snapshot.instances[0].demand).toEqual({ cpuCores: 2, memoryBytes: 2 * gib });
});

it('counts a pending migration once before and after ownership changes and never assumes unplaced pinning', async () => {
    const s = await setup();
    s.db()
        .prepare(
            `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
        VALUES('moving',?,'rebuild','waiting','Move',?,'test')`
        )
        .run(server, s.at);
    s.db()
        .prepare(
            'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)'
        )
        .run('moving', s.native.id);
    s.db()
        .prepare(
            `INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,target_host_id)
        VALUES('moving',?,?,?)`
        )
        .run(s.source.host.id, s.native.id, s.target.host.id);
    const target = () => s.read().hosts.find((host) => host.id === s.target.host.id)!;
    expect(target().used.cpuCores).toBe(0.5);
    expect(target().used.memoryBytes).toBe(2 * gib);
    expect(s.read('moving').hosts.find((host) => host.id === s.target.host.id)?.used.cpuCores).toBe(
        0
    );
    s.db().prepare('UPDATE native_instances SET host_id=?,generation=2').run(s.target.host.id);
    s.db().prepare('UPDATE native_provisioning SET generation=2').run();
    s.db().prepare('UPDATE ec_host_migrations SET removed_at=?').run(s.at);
    expect(target().used.cpuCores).toBe(0.5);
    expect(target().used.memoryBytes).toBe(2 * gib);
    s.db()
        .prepare(
            "UPDATE ec_hosts SET inventory=json_set(inventory,'$.instanceResources',json(?),'$.cpuUsagePercent',25,'$.memoryAvailable',?) WHERE id=?"
        )
        .run(JSON.stringify([{ ...s.resources, generation: 2 }]), 13 * gib, s.target.host.id);
    expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
    s.db()
        .prepare(
            `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
        VALUES('unplaced','t6_zombie_town_2','create','queued','Place',?,'test')`
        )
        .run(s.at);
    s.db()
        .prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)')
        .run('unplaced', JSON.stringify({ hostId: s.target.host.id }));
    expect(target().used).toEqual({ cpuCores: 2, memoryBytes: 3 * gib });
    expect(target().instances).toBe(1);
});

it('allows admission to ignore its own unsubmitted rebuild while retaining the source empty evidence', async () => {
    const s = await setup();
    s.db()
        .prepare(
            `INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
        VALUES('own',?,'rebuild','queued','Move',?,'rebalance:test')`
        )
        .run(server, s.at);
    expect(s.read().instances).toHaveLength(0);
    expect(s.read('own').instances).toHaveLength(1);
});

it('enforces the 30 minute cooldown across generations after a successful move', async () => {
    const s = await setup();
    s.db()
        .prepare(
            `INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message) VALUES('old','{}','done',?,?,'Done')`
        )
        .run(s.at, s.at);
    s.db()
        .prepare(
            `INSERT INTO rebalancing_moves(id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at)
        VALUES('m','old',0,?,?,0,?,?,'{}','done','Done',?)`
        )
        .run(server, s.native.id, s.target.host.id, s.source.host.id, s.at);
    expect(s.read().exclusions[0].reason).toContain('30 minutes');
    s.db()
        .prepare('UPDATE rebalancing_moves SET updated_at=?')
        .run(new Date(s.base - 30 * 60_000 - 1).toISOString());
    expect(s.read().instances).toHaveLength(1);
});

it('credits a freed game port only when the current range and bind will actually gain a port', async () => {
    const s = await setup();
    expect(s.read().instances[0].releasesPort).toBe(false);
    s.db()
        .prepare(
            `INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
        VALUES(?,?,'0.0.0.0','udp',28960,?)`
        )
        .run(s.source.host.id, s.native.id, s.at);
    expect(s.read().instances[0].releasesPort).toBe(true);
    s.db().prepare('UPDATE ec_hosts SET port_min=28970 WHERE id=?').run(s.source.host.id);
    expect(s.read().instances[0].releasesPort).toBe(false);
    s.db().prepare('UPDATE ec_hosts SET port_min=28960 WHERE id=?').run(s.source.host.id);
    s.db()
        .prepare(
            `INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at)
        VALUES(?,'other','127.0.0.2','tcp',28960,?)`
        )
        .run(s.source.host.id, s.at);
    expect(s.read().instances[0].releasesPort).toBe(false);
});