josh
981fb27c1c
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name. Fixes #16.
494 lines
28 KiB
TypeScript
494 lines
28 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
|
|
const healthyRcon = {
|
|
ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
|
|
fixture = await newFixture(servers);
|
|
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
|
vi.stubEnv('KOMODO_API_KEY', 'test');
|
|
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
|
vi.stubEnv('RCON_PASSWORD', 'test');
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
|
|
|
|
it('ignores a late restart reply after deletion without changing a replacement recovery slot', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { restartInRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
let complete!: (value: { ok: true }) => void;
|
|
vi.mocked(komodoRestart).mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));
|
|
const oldOperation = beginDeployment(servers[0], 'restart');
|
|
const pending = restartInRollout(servers[0], { operationId: oldOperation, action: 'restart' });
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
|
await seedServerRegistry();
|
|
const replacement = beginDeployment(servers[0], 'restart');
|
|
const requestedAt = new Date().toISOString();
|
|
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
|
|
.run(servers[0], requestedAt, replacement);
|
|
nextTick();
|
|
complete({ ok: true });
|
|
expect(await pending).toMatchObject({ kind: 'failed' });
|
|
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE id = ?').get(oldOperation))
|
|
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
|
expect(db().prepare('SELECT operation_id, requested_at, error FROM restart_rollout WHERE server = ?').get(servers[0]))
|
|
.toEqual({ operation_id: replacement, requested_at: requestedAt, error: null });
|
|
});
|
|
|
|
async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
|
|
.run(id, action, new Date().toISOString());
|
|
for (const server of targets) {
|
|
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
|
VALUES (?, ?, 1, 'pending')`).run(id, server);
|
|
}
|
|
}
|
|
|
|
async function rows(id = 'dep-1') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
|
|
.all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
|
|
}
|
|
|
|
async function beta(targets: string[], waitForHealth = true) {
|
|
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
|
|
return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
|
|
servers: targets, waitForHealth, waitForMatch: false });
|
|
}
|
|
|
|
describe('rolling deployment restarts', () => {
|
|
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect((await sweep()).restarted).toBe(1);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
|
|
await sweep(); // Repeated kick can't bypass the grace period.
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
|
expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
|
nextTick();
|
|
expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
|
|
expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
|
|
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
|
|
.toMatchObject({ finished_at: expect.any(String) });
|
|
});
|
|
|
|
it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
|
|
await deployment('z-oldest', [servers[2]], 'rollback');
|
|
nextTick();
|
|
await deployment('a-next', [servers[0]], 'drift');
|
|
nextTick();
|
|
await deployment('b-latest', [servers[1]], 'promote');
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
|
nextTick();
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
|
|
});
|
|
|
|
it('skips occupied and unreachable servers without consuming capacity', async () => {
|
|
await deployment();
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
|
});
|
|
|
|
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await sweep();
|
|
nextTick();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
let finish!: () => void;
|
|
const entered = new Promise<void>((resolve) => {
|
|
vi.mocked(komodoRestart).mockImplementationOnce(() => {
|
|
resolve();
|
|
return new Promise((done) => { finish = () => done({ ok: true }); });
|
|
});
|
|
});
|
|
const first = sweep();
|
|
await entered;
|
|
const second = sweep();
|
|
expect(second).toBe(first);
|
|
// Even a long-running restart request cannot be freed by a health poll.
|
|
nextTick();
|
|
const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
|
|
await refreshRestartRollout();
|
|
expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
|
|
finish();
|
|
await Promise.all([first, second]);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
|
await deployment('dep-1', [servers[0]]);
|
|
await deployment('dep-2', [servers[0], servers[1]]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
|
|
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
|
});
|
|
|
|
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
|
await deployment('dep-1', [servers[0]]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
await deployment('dep-2', [servers[0]]);
|
|
nextTick();
|
|
await sweep();
|
|
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
|
expect((await rows('dep-2'))[0].restart_state).toBe('pending');
|
|
// Provisioning can submit the next job in the pass that verifies the
|
|
// first, but it must issue a distinct restart and await its recovery.
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at);
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
|
});
|
|
|
|
it('preserves occupied slots across app/module restarts', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
await sweep();
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
vi.resetModules();
|
|
const fresh = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await fresh.sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await fresh.sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('adopts pre-upgrade restarts instead of starting a second server', async () => {
|
|
await deployment();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
|
.run(new Date().toISOString(), servers[0]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
|
await sweep();
|
|
expect((await rows())[0].restart_state).toBe('failed');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not restart a server whose file deployment failed', async () => {
|
|
await deployment();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
|
.run(servers[0]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
|
expect((await rows())[0].error).toBe('write failed');
|
|
});
|
|
|
|
it('releases slots for definite local failures that could not have started a restart', async () => {
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
|
await sweep();
|
|
expect((await rows())[0].restart_state).toBe('failed');
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
|
|
});
|
|
});
|
|
|
|
describe('beta and deployment coordination', () => {
|
|
it('limits beta restarts and releases capacity only after recovery', async () => {
|
|
await beta(servers);
|
|
const { betaTick } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const first = betaTick();
|
|
expect(betaTick()).toBe(first);
|
|
await first;
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await betaTick();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
|
await beta([servers[1]]);
|
|
await deployment('dep-1', [servers[0]]);
|
|
const { betaTick } = await import('../helpers/provisioning.js');
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await Promise.all([sweep(), betaTick()]);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
|
const { betaId } = await beta([servers[0]], false);
|
|
const { betaTick } = await import('../helpers/provisioning.js');
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await betaTick();
|
|
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
|
|
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
|
|
await deployment('dep-1', [servers[1]]);
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
nextTick();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('checks player occupancy again when resuming a legacy restarting beta', async () => {
|
|
const { betaId } = await beta([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
|
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
|
const { betaTick } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
|
await betaTick();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('Provisioning ownership', () => {
|
|
it('persists caller requests before probes and completes them without another caller tick', async () => {
|
|
await deployment('campaign', [servers[0]]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await sweep(); await sweep();
|
|
const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
|
|
const id = source.provisioning_id;
|
|
expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
|
|
expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0] }]);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
|
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 });
|
|
await processProvisioning(); nextTick(); await processProvisioning();
|
|
expect(provisioningOperation(id)?.state).toBe('done');
|
|
expect((await rows('campaign'))[0].restart_state).toBe('pending');
|
|
await sweep();
|
|
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('lets concurrent callers submit while only the owner admits restart capacity', async () => {
|
|
await deployment('campaign', [servers[0]]); await beta([servers[1]]);
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await Promise.all([sweep(), betaTick()]);
|
|
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
|
const first = processProvisioning();
|
|
expect(processProvisioning()).toBe(first);
|
|
await first;
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not replay an ambiguous campaign restart after recovery or app restart', async () => {
|
|
await deployment('campaign', [servers[0]]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
|
await sweep(); nextTick(); await sweep();
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb(); vi.resetModules();
|
|
const fresh = await import('../helpers/provisioning.js');
|
|
nextTick(); await fresh.sweep();
|
|
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('holds a paused beta request and cancels closed work without a beta runner tick', async () => {
|
|
const { betaId } = await beta([servers[0]]);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
await betaTick();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
await processProvisioning();
|
|
expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' });
|
|
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
|
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
|
|
await processProvisioning();
|
|
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' });
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => {
|
|
await deployment('campaign', [servers[0]]);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
|
await sweep(); nextTick(); await sweep();
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
|
expect(requestLifecycle(servers[0], 'restart').ok).toBe(true);
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
|
});
|
|
|
|
it('does not credit a legacy submission with an older completed recovery slot', async () => {
|
|
await deployment('campaign', [servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const old = new Date().toISOString(); nextTick();
|
|
const submitted = new Date().toISOString();
|
|
db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old);
|
|
db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted);
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' });
|
|
expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null });
|
|
nextTick(); await sweep();
|
|
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => {
|
|
const { betaId } = await beta([servers[0]]);
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
await betaTick();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string };
|
|
closeDb(); vi.resetModules();
|
|
const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js');
|
|
const source = { kind: 'beta' as const, id: String(betaId) };
|
|
expect(requestRestart(servers[0], source).id).toBe(previous.id);
|
|
releaseRestartRequest(servers[0], source);
|
|
const next = requestRestart(servers[0], source);
|
|
expect(next.id).not.toBe(previous.id);
|
|
expect(provisioningOperation(previous.id)?.state).toBe('cancelled');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect(provisioningOperation(next.id)?.state).toBe('awaiting-health');
|
|
});
|
|
});
|
|
|
|
describe('rollout configuration', () => {
|
|
it('honors a configured limit larger than one', async () => {
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
|
|
await deployment();
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('holds new restarts after lowering a live limit without releasing active slots', async () => {
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { sweep } = await import('../helpers/provisioning.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
|
|
await deployment();
|
|
await sweep();
|
|
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all()).toHaveLength(2);
|
|
nextTick();
|
|
await sweep();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(3);
|
|
});
|
|
});
|