electric-cherry/tests/unit/autoscaling-lifecycle.test.ts

import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { prepareNativeHost } from '../helpers/native-host.js';
import { gameEvent } from '../helpers/telemetry.js';
import { zombieMaps } from '../../src/lib/map-aliases.js';

let fixture: Fixture;
beforeEach(async () => {
    vi.resetModules();
    fixture = await newFixture([]);
});
afterEach(async () => {
    vi.useRealTimers();
    await fixture.cleanup();
});
const server = 't6_zombie_town_1';

async function setup(mapSlug: string) {
    const maps = zombieMaps('t6').filter((map) => mapSlug === '*' || map.slug === mapSlug);
    const names = maps.map((map) => `t6_zombie_${map.slug}_1`);
    const { host } = await prepareNativeHost(names);
    const { db } = await import('../../src/lib/server/db/index.js');
    const scaling = await import('../../src/lib/server/autoscaling/index.js');
    const { autoscalingRemovalCapability } =
        await import('../../src/lib/server/autoscaling/authorization.js');
    db()
        .prepare(
            "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
        )
        .run(JSON.stringify([autoscalingRemovalCapability]), host.id);
    const at = new Date().toISOString();
    for (const map of maps) {
        const name = `t6_zombie_${map.slug}_1`;
        db()
            .prepare(
                `INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
            VALUES(?,'t6',?,?,'running',?,?,?,0)`
            )
            .run(name, map.canonical, map.slug, at, at, at);
        db()
            .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
            .run(name, `key-${name}`);
        db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name);
    }
    // A free key ensures a lack of capacity cannot hide unwanted scale-up.
    db()
        .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')")
        .run();
    const policy = scaling.saveScaling({ game: 't6', mapSlug, maximum: 2, mode: 'create-destroy' });
    const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
    return { db, scaling, policy, requestLifecycle };
}

it.each(['town', '*'])('keeps lineup-changing actions blocked for %s ownership', async (scope) => {
    const { db, requestLifecycle } = await setup(scope);
    const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
    for (const action of ['start', 'stop'] as const) {
        expect(requestLifecycle(server, action)).toMatchObject({
            ok: false,
            status: 409,
            error: expect.stringContaining('Autoscaling')
        });
    }
    expect(destroyServer(server)).toMatchObject({
        ok: false,
        message: expect.stringContaining('Autoscaling')
    });
    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
});

it.each([
    ['town', 'restart'],
    ['town', 'rebuild'],
    ['*', 'restart'],
    ['*', 'rebuild']
] as const)(
    'allows %s-owned %s without scaling up during work or recovery',
    async (scope, action) => {
        vi.useFakeTimers({ toFake: ['Date'] });
        const { db, scaling, policy, requestLifecycle } = await setup(scope);
        const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
        const result = requestLifecycle(server, action);
        expect(result.ok).toBe(true);
        if (!result.ok) throw new Error(result.error);
        expect(requestLifecycle(server, action)).toEqual(result);

        for (const state of ['waiting', 'queued', 'running', 'awaiting-health', 'interrupted']) {
            db()
                .prepare('UPDATE server_deployments SET state=? WHERE id=?')
                .run(state, result.deploymentId);
            db()
                .prepare(
                    "UPDATE servers SET container_state='stopped',last_players=NULL WHERE name=?"
                )
                .run(server);
            const snapshot = scalingSnapshot(policy);
            expect(snapshot.members.find((member) => member.name === server)).toMatchObject({
                state: 'protected',
                operationId: result.deploymentId
            });
            expect(snapshot.reservedNames).toContain(server);
            expect(await scaling.runAutoscaling()).toBe(false);
        }

        // A terminal operation with unresolved recovery must still hold capacity.
        db()
            .prepare("UPDATE server_deployments SET state='failed' WHERE id=?")
            .run(result.deploymentId);
        const requestedAt = new Date().toISOString();
        db()
            .prepare('INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?)')
            .run(server, requestedAt, result.deploymentId);
        expect(
            scalingSnapshot(policy).members.find((member) => member.name === server)?.blocker
        ).toBe('Unresolved recovery');
        expect(await scaling.runAutoscaling()).toBe(false);
        expect(
            db()
                .prepare("SELECT id FROM server_deployments WHERE action IN ('create','destroy')")
                .all()
        ).toEqual([]);

        // Recovery status must be strictly newer than the restart request.
        const at = new Date(Date.parse(requestedAt) + 1_000).toISOString();
        vi.setSystemTime(new Date(at));
        db()
            .prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id=?")
            .run(at, result.deploymentId);
        db().prepare('UPDATE restart_rollout SET healthy_at=? WHERE server=?').run(at, server);
        db()
            .prepare(
                "UPDATE servers SET container_state='running',last_players=0,container_checked_at=?,last_seen_at=?,last_checked_at=?"
            )
            .run(at, at, at);
        expect(await scaling.runAutoscaling()).toBe(false);
        expect(scaling.getScalingPolicy(policy.id)).toMatchObject({
            mode: 'create-destroy',
            pausedReason: null
        });

        // Recovery and occupancy alone cannot request a spare before round two.
        db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(server);
        expect(await scaling.runAutoscaling()).toBe(false);
        vi.setSystemTime(Date.now() + 1_000);
        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
        saveTelemetry({
            ...gameEvent('heartbeat', 2, 1, server, 'zm_transit'),
            receivedAt: new Date().toISOString()
        });
        expect(await scaling.runAutoscaling()).toBe(true);
        expect(
            db().prepare("SELECT server FROM server_deployments WHERE action='create'").all()
        ).toEqual([{ server: 't6_zombie_town_2' }]);
    }
);