334 lines
17 KiB
TypeScript
334 lines
17 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
|
|
const healthyRcon = {
|
|
ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
|
|
fixture = await newFixture(servers);
|
|
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
|
vi.stubEnv('KOMODO_API_KEY', 'test');
|
|
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
|
vi.stubEnv('RCON_PASSWORD', 'test');
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
|
|
|
|
async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
|
|
.run(id, action, new Date().toISOString());
|
|
for (const server of targets) {
|
|
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
|
VALUES (?, ?, 1, 'pending')`).run(id, server);
|
|
}
|
|
}
|
|
|
|
async function rows(id = 'dep-1') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
|
|
.all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
|
|
}
|
|
|
|
async function beta(targets: string[], waitForHealth = true) {
|
|
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
|
|
return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
|
|
servers: targets, waitForHealth, waitForMatch: false });
|
|
}
|
|
|
|
describe('rolling deployment restarts', () => {
|
|
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect((await sweep()).restarted).toBe(1);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
|
|
await sweep(); // Repeated kick can't bypass the grace period.
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
|
expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
|
|
expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
|
|
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
|
|
.toMatchObject({ finished_at: expect.any(String) });
|
|
});
|
|
|
|
it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
|
|
await deployment('z-oldest', [servers[2]], 'rollback');
|
|
nextTick();
|
|
await deployment('a-next', [servers[0]], 'drift');
|
|
nextTick();
|
|
await deployment('b-latest', [servers[1]], 'promote');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
|
nextTick();
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
|
|
});
|
|
|
|
it('skips occupied and unreachable servers without consuming capacity', async () => {
|
|
await deployment();
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
|
});
|
|
|
|
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await sweep();
|
|
nextTick();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
let finish!: () => void;
|
|
const entered = new Promise<void>((resolve) => {
|
|
vi.mocked(komodoRestart).mockImplementationOnce(() => {
|
|
resolve();
|
|
return new Promise((done) => { finish = () => done({ ok: true }); });
|
|
});
|
|
});
|
|
const first = sweep();
|
|
await entered;
|
|
const second = sweep();
|
|
expect(second).toBe(first);
|
|
// Even a long-running restart request cannot be freed by a health poll.
|
|
nextTick();
|
|
const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
|
|
await refreshRestartRollout();
|
|
expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
|
|
finish();
|
|
await Promise.all([first, second]);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
|
await deployment('dep-1', [servers[0]]);
|
|
await deployment('dep-2', [servers[0], servers[1]]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
|
|
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
|
});
|
|
|
|
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
|
await deployment('dep-1', [servers[0]]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
await deployment('dep-2', [servers[0]]);
|
|
nextTick();
|
|
await sweep();
|
|
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
|
expect((await rows('dep-2'))[0].restarted_at).toBeNull();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
|
});
|
|
|
|
it('preserves occupied slots across app/module restarts', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
await sweep();
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
vi.resetModules();
|
|
const fresh = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await fresh.sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await fresh.sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('adopts pre-upgrade restarts instead of starting a second server', async () => {
|
|
await deployment();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
|
.run(new Date().toISOString(), servers[0]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
|
await sweep();
|
|
expect((await rows())[0].restart_state).toBe('failed');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not restart a server whose file deployment failed', async () => {
|
|
await deployment();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
|
.run(servers[0]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
|
expect((await rows())[0].error).toBe('write failed');
|
|
});
|
|
|
|
it('releases slots for definite local failures that could not have started a restart', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
|
await sweep();
|
|
expect((await rows())[0].restart_state).toBe('failed');
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
|
|
});
|
|
});
|
|
|
|
describe('beta and deployment coordination', () => {
|
|
it('limits beta restarts and releases capacity only after recovery', async () => {
|
|
await beta(servers);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const first = betaTick();
|
|
expect(betaTick()).toBe(first);
|
|
await first;
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await betaTick();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
|
await beta([servers[1]]);
|
|
await deployment('dep-1', [servers[0]]);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await Promise.all([sweep(), betaTick()]);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
|
const { betaId } = await beta([servers[0]], false);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await betaTick();
|
|
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
|
|
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
|
|
await deployment('dep-1', [servers[1]]);
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('checks player occupancy again when resuming a legacy restarting beta', async () => {
|
|
const { betaId } = await beta([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
|
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
|
await betaTick();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('rollout configuration', () => {
|
|
it('honors a configured limit larger than one', async () => {
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
vi.resetModules();
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
|
await deployment();
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => {
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
vi.resetModules();
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value);
|
|
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/);
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
vi.resetModules();
|
|
});
|
|
});
|