Remove Komodo backend in favor of native hosts
This commit is contained in:
@@ -1,17 +1,9 @@
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { access } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeploy: vi.fn(async () => ({ ok: true })),
|
||||
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
@@ -24,19 +16,18 @@ const server = 't5_zombie_theater_1';
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([otherServer, server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
|
||||
KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28980' })) vi.stubEnv(key, value);
|
||||
vi.stubEnv('RCON_PASSWORD', 'test');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const [index, name] of [otherServer, server].entries()) {
|
||||
const map = name === server ? 'theater' : 'cosmodrome';
|
||||
db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name));
|
||||
db().prepare(`INSERT INTO servers (name, game, map, game_mode, map_slug, game_port, host_dir)
|
||||
VALUES (?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name));
|
||||
}
|
||||
await prepareNativeHost([otherServer, server]);
|
||||
const { mockAgentFiles } = await import('../helpers/agent-files.js');
|
||||
await mockAgentFiles();
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0);
|
||||
vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
@@ -55,9 +46,7 @@ async function attachBeta(state = 'running', outcome?: string, deleted = false)
|
||||
|
||||
async function expectServerPreserved() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toEqual({ name: server });
|
||||
await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
|
||||
}
|
||||
@@ -117,86 +106,6 @@ describe('active beta destruction guard', () => {
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true, deploymentId: expect.any(String) });
|
||||
});
|
||||
|
||||
it('holds a queued destruction until its beta closes, then completes the same request', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = destroyServer(server);
|
||||
expect(queued.ok).toBe(true);
|
||||
const id = await attachBeta();
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
await expectServerPreserved();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(new Date().toISOString(), id);
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledOnce();
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledOnce();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
});
|
||||
|
||||
it('rechecks beta ownership after an in-flight empty-server probe', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
let resolveCount!: (players: number) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
|
||||
const queued = destroyServer(server);
|
||||
const work = processDestructions();
|
||||
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
|
||||
await attachBeta();
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('also holds rebuild destruction when a beta appears during the occupancy probe', async () => {
|
||||
const { requestLifecycle, processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
let resolveCount!: (players: number) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
|
||||
const queued = requestLifecycle(server, 'rebuild');
|
||||
if (!queued.ok) throw new Error(queued.error);
|
||||
const work = processLifecycle();
|
||||
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
|
||||
await attachBeta();
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('preserves the cleanup checkpoint and files while an active beta blocks a legacy cleanup resume', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { deploymentStage, finishDeployment, resumeCleanup, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const queued = destroyServer(server);
|
||||
deploymentStage(queued.deploymentId!, REMOVING_SERVER);
|
||||
finishDeployment(queued.deploymentId!, { ok: false, message: 'Interrupted local cleanup' });
|
||||
const id = await attachBeta();
|
||||
expect(resumeCleanup(queued.deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER,
|
||||
error: expect.stringMatching(/beta/i) });
|
||||
expect(getDeployment(queued.deploymentId!)?.canResumeCleanup).toBe(true);
|
||||
await expectServerPreserved();
|
||||
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(id);
|
||||
expect(resumeCleanup(queued.deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['destroy', 'rebuild'] as const)('requires cancelling a pending %s before opening a beta', async action => {
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
@@ -222,7 +131,7 @@ describe('active beta destruction guard', () => {
|
||||
});
|
||||
|
||||
it.each((['destroy', 'rebuild'] as const).flatMap(action => ['failed', 'interrupted'].flatMap(state =>
|
||||
['Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'].map(stage => ({ action, state, stage }))
|
||||
['Stopping and removing container', 'Removing server files and registry'].map(stage => ({ action, state, stage }))
|
||||
)))('rejects beta admission after $action is $state at $stage', async ({ action, state, stage }) => {
|
||||
const { beginDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
|
||||
@@ -6,9 +6,26 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
|
||||
import { defaultPlan, requirement, type BetaPlan } from '../../src/lib/beta-plan.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
|
||||
// Planner tests observe the provisioning contract. Host command delivery and
|
||||
// disruption safety are exercised by the native provisioning suites.
|
||||
vi.mock('../../src/lib/server/provisioning/worker.js', () => ({
|
||||
processProvisioning: async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { probePlayerEvidence } = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
const jobs = db().prepare("SELECT id,server,state FROM server_deployments WHERE action='restart' AND state IN ('waiting','awaiting-health')").all() as { id: string; server: string; state: string }[];
|
||||
for (const job of jobs) {
|
||||
if (job.state === 'awaiting-health') {
|
||||
db().prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
} else if ((await probePlayerEvidence(job.server)).players === 0) {
|
||||
applyStagedFilesForOperation(job.id, job.server);
|
||||
db().prepare("UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
let fixture: Fixture;
|
||||
const names = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
|
||||
const now = () => new Date().toISOString();
|
||||
@@ -21,14 +38,11 @@ beforeEach(async () => {
|
||||
await writeFile(join(fixture.t5, name, 'logs/console.log'), 'old script error\n');
|
||||
await writeFile(join(fixture.t5, name, 'logs/games_sp.log'), 'InitGame:\n 0:00 J;old;1;old\nExitLevel: executed\n');
|
||||
}
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 1 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||
});
|
||||
@@ -57,14 +71,12 @@ describe('durable beta test plans', () => {
|
||||
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
|
||||
const id = await openPlan(defaultPlan());
|
||||
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await postTelemetry(gameEvent('heartbeat', 4, 2));
|
||||
await tick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(listBetaServers(id)[0].stage).toBe('awaiting-empty');
|
||||
await postTelemetry(gameEvent('heartbeat', 4, 0));
|
||||
await tick();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(listBetaServers(id)[0].stage).toBe('awaiting-health');
|
||||
});
|
||||
|
||||
it('holds file deployment until each wave passes, and rollback leaves undeployed files alone', async () => {
|
||||
|
||||
@@ -9,7 +9,7 @@ let fixture: Fixture;
|
||||
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', '');
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
|
||||
@@ -188,7 +188,7 @@ describe('staged beta files', () => {
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId });
|
||||
const second = requestRestart(servers[1], { kind: 'campaign', id: latest.deploymentId });
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(second.id).not.toBe(first.id);
|
||||
await activateFiles('campaign', latest.deploymentId);
|
||||
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
||||
const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]);
|
||||
|
||||
@@ -18,9 +18,7 @@ beforeEach(async () => {
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image);
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('KOMODO_ADDRESS', '');
|
||||
vi.stubEnv('KOMODO_API_KEY', '');
|
||||
vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Large native fleet', portMin: 28960, portMax: 29059 }).id;
|
||||
const inventory = {
|
||||
|
||||
@@ -24,9 +24,7 @@ beforeEach(async () => {
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image);
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('KOMODO_ADDRESS', '');
|
||||
vi.stubEnv('KOMODO_API_KEY', '');
|
||||
vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id;
|
||||
@@ -55,8 +53,8 @@ beforeEach(async () => {
|
||||
const instanceId = randomUUID();
|
||||
const deploymentId = randomUUID();
|
||||
const port = 28960 + index;
|
||||
db().prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at)
|
||||
VALUES(?,'','t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now);
|
||||
db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at)
|
||||
VALUES(?,'t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now);
|
||||
db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)')
|
||||
.run(instanceId, name, hostId, libraries[1].id, image, port, now);
|
||||
db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)")
|
||||
|
||||
@@ -10,7 +10,7 @@ let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(['t5_zombie_theater_1']);
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
|
||||
|
||||
@@ -1,803 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { readFile, rm } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { T6_ZOMBIE_MAPS } from '../../src/lib/map-aliases.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeploy: vi.fn(async () => ({ ok: true })),
|
||||
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const serverName = 't5_zombie_cosmodrome_1';
|
||||
const input = { gameMode: 'zombie' as const, mapSlug: 'cosmodrome', plutoKey: 'test-key' };
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-rcon');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult, rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockReset().mockResolvedValue({ ok: true,
|
||||
endpoint: { host: 'localhost', port: 27016 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } });
|
||||
vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('queued removal cancellation', () => {
|
||||
it('lets cancellation win while the worker awaits a player count, and never removes the server', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
let resolveCount!: (players: number) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
|
||||
const queued = destroyServer(serverName);
|
||||
const work = processDestructions();
|
||||
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
|
||||
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200);
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Cancelled by operator' });
|
||||
expect(getDeployment(queued.deploymentId!)?.canCancelRemoval).toBe(false);
|
||||
expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200);
|
||||
});
|
||||
|
||||
it('does not overwrite cancellation when the in-flight player check rejects', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { cancelRemoval, getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await createServer(input);
|
||||
let rejectCount!: (error: Error) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectCount = reject; }));
|
||||
const queued = destroyServer(serverName);
|
||||
const work = processDestructions();
|
||||
await vi.waitFor(() => expect(rejectCount).toBeTypeOf('function'));
|
||||
expect(cancelRemoval(queued.deploymentId!)).toBe(true);
|
||||
rejectCount(new Error('RCON unavailable'));
|
||||
await work;
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', error: null });
|
||||
});
|
||||
|
||||
it('refuses cancellation after an atomic removal claim and for cleanup checkpoints', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { cancelRemoval, claimRemoval, canCancelRemoval, deploymentStage, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await createServer(input);
|
||||
const queued = destroyServer(serverName);
|
||||
expect(claimRemoval(queued.deploymentId!)).toBe(true);
|
||||
expect(cancelRemoval(queued.deploymentId!)).toBe(false);
|
||||
expect(claimRemoval(queued.deploymentId!)).toBe(false);
|
||||
deploymentStage(queued.deploymentId!, REMOVING_SERVER);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(queued.deploymentId);
|
||||
expect(canCancelRemoval(queued.deploymentId!)).toBe(false);
|
||||
expect(cancelRemoval(queued.deploymentId!)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('destruction cleanup recovery', () => {
|
||||
it('removes T6 servers while preserving closed beta progress and test evidence', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true,
|
||||
endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } });
|
||||
const name = 't6_zombie_town_1';
|
||||
expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true });
|
||||
const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') });
|
||||
const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'promoted')").run(script.id, version.id).lastInsertRowid;
|
||||
for (const server of [name, serverName]) {
|
||||
db().prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (?, ?, 0, 'ready', '2026-09-18')").run(beta, server);
|
||||
db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server);
|
||||
db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server);
|
||||
db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server);
|
||||
db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server);
|
||||
}
|
||||
const queued = destroyServer(name);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
for (const table of ['beta_servers', 'beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
|
||||
expect(db().prepare(`SELECT server FROM ${table} ORDER BY server`).all()).toEqual([{ server: serverName }, { server: name }]);
|
||||
}
|
||||
expect(db().prepare('SELECT run_id FROM beta_test_evidence ORDER BY run_id').all()).toEqual([{ run_id: serverName }, { run_id: name }]);
|
||||
expect(db().prepare('SELECT stage, healthy_at, server_deleted_at FROM beta_servers WHERE server = ?').get(name))
|
||||
.toEqual({ stage: 'ready', healthy_at: '2026-09-18', server_deleted_at: expect.any(String) });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
});
|
||||
|
||||
it('resumes failed local cleanup after restart without RCON or another Komodo deletion', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { POST } = await import('../../src/routes/api/deployments/[id]/resume/+server.js');
|
||||
await createServer(input);
|
||||
db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'FOREIGN KEY constraint failed'); END");
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
await processDestructions();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: 'Removing server files and registry', server_exists: 1 });
|
||||
expect(getDeployment(deploymentId!)?.canResumeCleanup).toBe(true);
|
||||
db().exec('DROP TRIGGER fail_cleanup');
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
vi.mocked(rconGetPlayerCount).mockClear();
|
||||
vi.mocked(komodoDestroyDeployment).mockClear();
|
||||
vi.mocked(komodoDeleteDeployment).mockClear();
|
||||
expect((await POST({ params: { id: deploymentId! } })).status).toBe(202);
|
||||
expect((await POST({ params: { id: deploymentId! } })).status).toBe(409);
|
||||
await processDestructions();
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
});
|
||||
|
||||
it('reuses an interrupted cleanup when destruction is requested again', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { deploymentStage, interruptDeployments, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await createServer(input);
|
||||
const first = destroyServer(serverName);
|
||||
deploymentStage(first.deploymentId!, REMOVING_SERVER);
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(first.deploymentId);
|
||||
interruptDeployments();
|
||||
expect(destroyServer(serverName).deploymentId).toBe(first.deploymentId);
|
||||
expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
|
||||
});
|
||||
|
||||
it('resumes cleanup after crashing between registry deletion and the final result', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { claimRemoval, deploymentStage, interruptDeployments, getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
claimRemoval(deploymentId!);
|
||||
deploymentStage(deploymentId!, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(serverName);
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'interrupted', server_deleted_at: expect.any(String) });
|
||||
expect(resumeCleanup(deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_deleted_at: expect.any(String), server_exists: 0 });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['before', 'after'] as const)('protects a replacement registered %s an archived cleanup resume', async timing => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { claimRemoval, deploymentStage, interruptDeployments, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await createServer(input);
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
claimRemoval(deploymentId!);
|
||||
deploymentStage(deploymentId!, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(serverName);
|
||||
interruptDeployments();
|
||||
if (timing === 'after') expect(resumeCleanup(deploymentId!)).toBe(true);
|
||||
db().prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(serverName, 'replacement');
|
||||
if (timing === 'before') expect(resumeCleanup(deploymentId!)).toBe(false);
|
||||
await processDestructions();
|
||||
expect(db().prepare('SELECT komodo_name FROM servers WHERE name = ?').get(serverName)).toEqual({ komodo_name: 'replacement' });
|
||||
});
|
||||
|
||||
it('refuses cleanup before confirmed deletion or after a newer creation', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
await createServer(input);
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
finishDeployment(deploymentId!, { ok: false, message: 'Komodo unavailable' });
|
||||
expect(resumeCleanup(deploymentId!)).toBe(false);
|
||||
deploymentStage(deploymentId!, REMOVING_SERVER);
|
||||
beginDeployment(serverName, 'create');
|
||||
expect(resumeCleanup(deploymentId!)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('House Party Protocol', () => {
|
||||
it('includes registered and queued servers once and reuses destruction requests', async () => {
|
||||
const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
|
||||
expect(await createServer(input)).toMatchObject({ ok: true });
|
||||
expect(queueCreateServer(input)).toMatchObject({ ok: true });
|
||||
const names = housePartyTargets();
|
||||
expect(names).toEqual([serverName, 't5_zombie_cosmodrome_2']);
|
||||
const first = queueHouseParty(names);
|
||||
expect(first).toMatchObject({ ok: true, deployments: [{ server: serverName }, { server: names[1] }] });
|
||||
expect(queueHouseParty(names)).toEqual(first);
|
||||
});
|
||||
|
||||
it('rejects a changed fleet without queuing any destruction', async () => {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const names = housePartyTargets();
|
||||
expect(queueCreateServer(input).ok).toBe(true);
|
||||
expect(queueHouseParty(names).ok).toBe(false);
|
||||
expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('rolls back the entire bulk request when an insert fails', async () => {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(queueCreateServer(input).ok).toBe(true);
|
||||
expect(queueCreateServer(input).ok).toBe(true);
|
||||
db().exec(`CREATE TRIGGER reject_destroy BEFORE INSERT ON server_deployments
|
||||
WHEN NEW.action = 'destroy' AND NEW.server = 't5_zombie_cosmodrome_2'
|
||||
BEGIN SELECT RAISE(ABORT, 'test failure'); END`);
|
||||
expect(() => queueHouseParty(housePartyTargets())).toThrow('test failure');
|
||||
expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('waits for queued creations, then for verified emptiness, and preserves keys', async () => {
|
||||
const { queueCreateServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
expect(queueCreateServer(input).ok).toBe(true);
|
||||
expect(queueHouseParty(housePartyTargets()).ok).toBe(true);
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
await processCreations();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2);
|
||||
await processDestructions();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null);
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never);
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledWith(serverName);
|
||||
expect(getServer(serverName)).toBeUndefined();
|
||||
expect(getServerKey(serverName)).toBe('test-key');
|
||||
});
|
||||
});
|
||||
|
||||
async function promotedConfig() {
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const artifact of [
|
||||
{ name: 'baseline', kind: 'baseline' as const, scopeType: 'game' as const, scopeValue: 't5',
|
||||
content: Buffer.from('set sv_voice "1"\nset sv_maprotation "map zombie_theater"\n') },
|
||||
{ name: 'map settings', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome',
|
||||
content: Buffer.from('set sv_maprotation "map zombie_moon"\nset g_inactivity "0"\n') },
|
||||
{ name: 'other map', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_theater',
|
||||
content: Buffer.from('set sv_voice "0"\n') }
|
||||
]) {
|
||||
const { script } = createScript(artifact);
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?')
|
||||
.run(getLatestVersion(script.id)!.id, script.id);
|
||||
}
|
||||
}
|
||||
|
||||
describe('T6 Zombies provisioning', () => {
|
||||
it('keeps T6 storage and config overrides separate from T5 settings', async () => {
|
||||
vi.stubEnv('T6_SERVERS_ROOT', join(fixture.root, 'custom-t6'));
|
||||
vi.stubEnv('T6_CONFIG_TARGET_FILE', 'custom_zm.cfg');
|
||||
vi.stubEnv('T6_REL_DEST_SCRIPT', 'custom/scripts/zm');
|
||||
vi.stubEnv('PLUTAINER_T6_GAMEFILES_VOLUME', 'bo2-files');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const baseline = createScript({ name: 'T6 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't6', content: Buffer.from('sv_voice 1\n') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id);
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { destOf, configFileOf } = await import('../../src/lib/server/config.js');
|
||||
const created = await createServer({ ...input, game: 't6', mapSlug: 'town' });
|
||||
if (!created.ok) throw new Error(created.message);
|
||||
const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0];
|
||||
expect(spec.volumes).toContain('bo2-files:/home/plutainer/gamefiles:ro');
|
||||
expect(spec.environment).toContainEqual({ variable: 'PLUTAINER_CONFIG_FILE', value: 'custom_zm.cfg' });
|
||||
expect(existsSync(join(fixture.root, 'custom-t6', created.name, 'configs/custom_zm.cfg'))).toBe(true);
|
||||
expect(destOf(created.name)).toBe(`${join(fixture.root, 'custom-t6')}/${created.name}/custom/scripts/zm`);
|
||||
expect(configFileOf(serverName)).toBe('dedicated_sp.cfg');
|
||||
expect(destOf(serverName)).toBe(`${fixture.t5}/${serverName}/runtime/plutonium/storage/t5/raw/scripts/sp`);
|
||||
});
|
||||
|
||||
it.each(T6_ZOMBIE_MAPS)('provisions $friendly with the T6 mount, config, and ruleset', async (map) => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const name = `t6_zombie_${map.slug}_1`;
|
||||
expect(await createServer({ ...input, game: 't6', mapSlug: map.slug })).toMatchObject({ ok: true, name });
|
||||
const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0];
|
||||
const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value]));
|
||||
expect(env).toMatchObject({ PLUTAINER_GAME: 't6zm', PLUTAINER_CONFIG_FILE: 'dedicated_zm.cfg',
|
||||
PLUTAINER_RCON_PASSWORD: 'test-rcon', PLUTAINER_MAP_ROTATE: 'false',
|
||||
PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${name} +map_rotate` });
|
||||
expect(env.PLUTAINER_SKIP_SEED).toBeUndefined();
|
||||
expect(spec.volumes).toEqual([`${fixture.t6}/${name}:/home/plutainer/app`, 't6_game_files:/home/plutainer/gamefiles:ro']);
|
||||
expect(getServer(name)).toMatchObject({ game: 't6', game_id: 't6zm', map: map.canonical,
|
||||
map_slug: map.slug, config_file: 'dedicated_zm.cfg' });
|
||||
expect(await readFile(join(fixture.t6, name, 'configs/electric_cherry_map.cfg'), 'utf8'))
|
||||
.toContain(`set sv_maprotation "execgts ${map.gameSettings} map ${map.canonical}"`);
|
||||
expect(existsSync(join(fixture.t5, name))).toBe(false);
|
||||
expect(existsSync(join(fixture.t6, name, 'configs/dedicated_zm.cfg'))).toBe(false);
|
||||
});
|
||||
|
||||
it('persists T6 queued work, reuses its key, and allocates ports across both games', async () => {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveServerKey, validServerId } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
saveServerKey('t6_zombie_town_1', 't6-key');
|
||||
expect(validServerId('t6_zombie_town_20')).toBe(true);
|
||||
expect(validServerId('t5_zombie_town_1')).toBe(false);
|
||||
expect(validServerId('t6_zombie_theater_1')).toBe(false);
|
||||
const t5 = queueCreateServer(input);
|
||||
const t6 = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' });
|
||||
if (!t5.ok || !t6.ok) throw new Error('queue failed');
|
||||
expect(t6.port).not.toBe(t5.port);
|
||||
closeDb();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockImplementation(async (server) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : 'zombie_cosmodrome', gametype: null, roster: [] } }));
|
||||
await processCreations();
|
||||
expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
await processCreations();
|
||||
expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
await processCreations();
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
const content = await readFile(join(fixture.t6, t6.name, 'configs/electric_cherry_map.cfg'), 'utf8');
|
||||
expect(content).toContain('execgts zm_standard_town.cfg map zm_transit');
|
||||
});
|
||||
|
||||
it('installs only matching promoted artifacts into T6 paths and removes only its directory on destruction', async () => {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const game of ['t5', 't6']) {
|
||||
for (const kind of ['script', 'baseline'] as const) {
|
||||
const { script, version } = createScript({ name: `${game}-${kind}`, kind, scopeType: 'game', scopeValue: game,
|
||||
targetRelPath: kind === 'script' ? `${game}.gsc` : undefined,
|
||||
content: Buffer.from(kind === 'script' ? 'main(){}' : 'sv_voice 1\nsv_maprotation "map wrong"\n') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
}
|
||||
}
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
|
||||
await createServer(input);
|
||||
const t6 = await createServer({ ...input, game: 't6', mapSlug: 'tomb' });
|
||||
if (!t6.ok) throw new Error(t6.message);
|
||||
expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t6.gsc'))).toBe(true);
|
||||
expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t5.gsc'))).toBe(false);
|
||||
expect(await readFile(join(fixture.t6, t6.name, 'configs/dedicated_zm.cfg'), 'utf8'))
|
||||
.toContain('execgts zm_classic_tomb.cfg map zm_tomb');
|
||||
expect(await scanServerDrift(t6.name)).toEqual([]);
|
||||
expect(destroyServer(t6.name).ok).toBe(true);
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(existsSync(join(fixture.t6, t6.name))).toBe(false);
|
||||
expect(existsSync(join(fixture.t5, serverName))).toBe(true);
|
||||
expect(getServerKey(t6.name)).toBe('test-key');
|
||||
});
|
||||
|
||||
it('validates game/map combinations at the API boundary', async () => {
|
||||
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
||||
for (const game of ['t7', null, 6]) {
|
||||
const res = await POST({ request: new Request('http://localhost/api/servers', {
|
||||
method: 'POST', body: JSON.stringify({ ...input, game })
|
||||
}) });
|
||||
expect(res.status).toBe(400);
|
||||
}
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
expect(queueCreateServer({ ...input, game: 't6' })).toMatchObject({ ok: false, kind: 'unknown-map' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('server creation map selection', () => {
|
||||
it('returns a queued creation before provisioning and reserves its name and port', async () => {
|
||||
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const response = await POST({ request: new Request('http://localhost/api/servers', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input)
|
||||
}) });
|
||||
expect(response.status).toBe(202);
|
||||
const body = await response.json();
|
||||
expect(body).toMatchObject({ name: serverName, deploymentId: expect.any(String) });
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued', expected_map: 'zombie_cosmodrome' });
|
||||
expect(queueCreateServer({ ...input, expectedName: serverName })).toMatchObject({ ok: false, kind: 'no-slot' });
|
||||
const second = queueCreateServer(input);
|
||||
expect(second).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' });
|
||||
if (second.ok) expect(second.port).not.toBe(body.port);
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
await Promise.all([processCreations(), processCreations()]);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null });
|
||||
if (second.ok) expect(getDeployment(second.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('completes creation only when Komodo, RCON, and the selected map all pass', async () => {
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const created = await createServer(input);
|
||||
if (!created.ok) throw new Error('fixture creation failed');
|
||||
const id = created.deploymentId;
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await processCreations();
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' });
|
||||
await processCreations();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON or fresh telemetry health: timeout' });
|
||||
for (const map of [null, 'zombie_moon']) {
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
if (created.port === null) throw new Error('A provisioned legacy server must have a port.');
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } });
|
||||
await processCreations();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') });
|
||||
}
|
||||
interruptDeployments();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, map zombie_cosmodrome, and Plutonium r5354 (production r5354)' });
|
||||
});
|
||||
|
||||
it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
|
||||
const { GET } = await import('../../src/routes/api/deployments/[id]/+server.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
const response = await DELETE({ params: { name: serverName } });
|
||||
expect(response.status).toBe(202);
|
||||
const body = await response.json();
|
||||
expect(body).toMatchObject({ ok: true, deploymentId: expect.any(String) });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
const detail = await (await GET({ params: { id: body.deploymentId } })).json();
|
||||
expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' });
|
||||
expect(detail.events.map((event: { message: string }) => event.message)).toEqual(expect.arrayContaining([
|
||||
'Server destruction requested', 'Waiting to verify the server is empty'
|
||||
]));
|
||||
expect(detail.events.every((event: { id: number }, index: number) => index === 0 || detail.events[index - 1].id < event.id)).toBe(true);
|
||||
const missing = await DELETE({ params: { name: 'missing-server' } });
|
||||
expect(missing.status).toBe(404);
|
||||
});
|
||||
|
||||
it('queues destruction, waits for live zero players, and preserves an ordered timeline', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await createServer(input);
|
||||
const queued = destroyServer(serverName);
|
||||
expect(queued).toMatchObject({ ok: true, deploymentId: expect.any(String) });
|
||||
expect(destroyServer(serverName).deploymentId).toBe(queued.deploymentId);
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3).mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('RCON unavailable'));
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' });
|
||||
}
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await Promise.all([processDestructions(), processDestructions()]);
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
const detail = getDeployment(queued.deploymentId!);
|
||||
expect(detail?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
const messages = (detail?.events as { message: string }[]).map(e => e.message);
|
||||
expect(messages).toEqual(expect.arrayContaining(['Server destruction requested', 'Waiting for server to empty: 3 player(s) online', 'Server is empty (0 players verified)', 'Stopping and removing container', 'Completed']));
|
||||
expect(messages.indexOf('Server is empty (0 players verified)')).toBeLessThan(messages.indexOf('Stopping and removing container'));
|
||||
});
|
||||
|
||||
it('preserves files and registry when Komodo rejects destruction and exposes failure details', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
await createServer(input);
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'denied' });
|
||||
const queued = destroyServer(serverName);
|
||||
await processDestructions();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(existsSync(join(fixture.t5, serverName))).toBe(true);
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', server_exists: 1, error: 'DestroyDeployment failed: denied' });
|
||||
});
|
||||
|
||||
it('records in-flight creation and retains both operations after destruction', async () => {
|
||||
const { createServer, destroyServer, processDestructions, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockImplementationOnce(async () => {
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ server: serverName, action: 'create', state: 'running', stage: 'Starting container' }));
|
||||
return { ok: true };
|
||||
});
|
||||
expect((await createServer(input)).ok).toBe(true);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => {
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ action: 'destroy', state: 'running' }));
|
||||
return { ok: true };
|
||||
});
|
||||
expect((await destroyServer(serverName)).ok).toBe(true);
|
||||
await processDestructions();
|
||||
expect(listDeployments()).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ server: serverName, action: 'create', state: 'done', server_exists: 0 }),
|
||||
expect.objectContaining({ server: serverName, action: 'destroy', state: 'done', server_exists: 0 })
|
||||
]));
|
||||
});
|
||||
|
||||
it('records failed and unexpectedly interrupted creation attempts', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { listDeployments, beginDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' });
|
||||
expect((await createServer(input)).ok).toBe(false);
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' }));
|
||||
vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost'));
|
||||
await expect(createServer(input)).rejects.toThrow('connection lost');
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'awaiting-health',
|
||||
stage: 'Verifying existing server after interrupted startup', finished_at: null }));
|
||||
const id = beginDeployment('interrupted-server', 'destroy');
|
||||
interruptDeployments();
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) }));
|
||||
});
|
||||
|
||||
it('installs only promoted scope-matching scripts and plugins before first boot', async () => {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind } = await import('../../src/lib/server/config.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const artifacts = [
|
||||
{ kind: 'script' as const, target: 'zm_spawn_fix.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: true, expected: true },
|
||||
{ kind: 'plugin' as const, target: 'plugin.dll', scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', promoted: true, expected: true },
|
||||
{ kind: 'script' as const, target: 'other.gsc', scopeType: 'map' as const, scopeValue: 'zombie_theater', promoted: true, expected: false },
|
||||
{ kind: 'script' as const, target: 'draft.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: false, expected: false }
|
||||
].map((a) => {
|
||||
const { script, version } = createScript({ name: a.target, kind: a.kind, targetRelPath: a.target,
|
||||
scopeType: a.scopeType, scopeValue: a.scopeValue, content: Buffer.from(a.target) });
|
||||
if (a.promoted) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
return { ...a, version };
|
||||
});
|
||||
vi.mocked(komodoDeploy).mockImplementationOnce(async () => {
|
||||
for (const a of artifacts) {
|
||||
const path = join(destOfKind(serverName, a.kind), a.target);
|
||||
expect(existsSync(path)).toBe(a.expected);
|
||||
if (a.expected) {
|
||||
expect(await readFile(path)).toEqual(a.version.content);
|
||||
expect(getServerScript(serverName, a.target)).toMatchObject({ script_version_id: a.version.id, source: 'create' });
|
||||
}
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
expect(await createServer(input)).toMatchObject({ ok: true });
|
||||
expect(await scanServerDrift(serverName)).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps Plutainer defaults without a baseline and overrides the map before rotation', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const result = await createServer(input);
|
||||
expect(result).toMatchObject({ ok: true, name: serverName });
|
||||
expect(existsSync(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'))).toBe(false);
|
||||
const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0];
|
||||
const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value]));
|
||||
expect(env.PLUTAINER_MAP_ROTATE).toBe('false');
|
||||
expect(env.PLUTAINER_EXTRA_ARGS).toBe(`+exec electric_cherry_map.cfg +set tws_server_id ${serverName} +map_rotate`);
|
||||
expect(await readFile(join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'), 'utf8'))
|
||||
.toContain('set sv_maprotation "map zombie_cosmodrome"');
|
||||
expect(env.PLUTAINER_CONFIG_FILE).toBe('dedicated_sp.cfg');
|
||||
expect(env.PLUTAINER_SKIP_SEED).toBeUndefined();
|
||||
expect(komodoDeploy).toHaveBeenCalledWith(serverName);
|
||||
expect(getServer(serverName)?.map).toBe('zombie_cosmodrome');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
expect(await scanServerDrift(serverName)).toEqual([]);
|
||||
const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg');
|
||||
await rm(mapPath);
|
||||
const repair = await runDriftDeployment({ servers: [serverName] });
|
||||
expect(repair.totalRestored).toBe(1);
|
||||
expect(existsSync(mapPath)).toBe(false);
|
||||
const { activateFiles } = await import('../helpers/staged-files.js');
|
||||
await activateFiles('campaign', repair.deploymentId!);
|
||||
expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"');
|
||||
expect(await scanServerDrift(serverName)).toEqual([]);
|
||||
});
|
||||
|
||||
it('writes promoted settings and selected map before first boot and tracks the file', async () => {
|
||||
await promotedConfig();
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
|
||||
vi.mocked(komodoDeploy).mockImplementationOnce(async () => {
|
||||
const config = await readFile(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'), 'utf8');
|
||||
expect(config).toContain('set sv_maprotation "map zombie_cosmodrome"');
|
||||
expect(config).toContain('set sv_voice "1"');
|
||||
expect(config).toContain('set g_inactivity "0"');
|
||||
expect(config).not.toContain('map zombie_moon');
|
||||
expect(getServerScript(serverName, 'dedicated_sp.cfg')?.source).toBe('create');
|
||||
return { ok: true };
|
||||
});
|
||||
expect(await createServer(input)).toMatchObject({ ok: true });
|
||||
expect(await scanServerDrift(serverName)).toEqual([]);
|
||||
});
|
||||
|
||||
it('removes the generated config and inventory if deployment fails', async () => {
|
||||
await promotedConfig();
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { script, version } = createScript({ name: 'spawn fix', kind: 'script', scopeType: 'game', scopeValue: 't5',
|
||||
targetRelPath: 'zm_spawn_fix.gsc', content: Buffer.from('main(){}') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' });
|
||||
expect(await createServer(input)).toMatchObject({ ok: false, kind: 'komodo' });
|
||||
expect(getServer(serverName)).toBeUndefined();
|
||||
expect(getServerScript(serverName, 'dedicated_sp.cfg')).toBeUndefined();
|
||||
expect(getServerScript(serverName, 'zm_spawn_fix.gsc')).toBeUndefined();
|
||||
expect(existsSync(join(fixture.t5, serverName))).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects unknown maps before creating a deployment', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await createServer({ ...input, mapSlug: 'unknown' })).toMatchObject({ ok: false, kind: 'unknown-map' });
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('persistent server keys', () => {
|
||||
it('saves a key, retains it through destruction, and reuses it for the exact lobby', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const name = 't5_zombie_cosmodrome_2';
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: serverName });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name });
|
||||
expect(getServerKey(name)).toBe('test-key');
|
||||
expect(await destroyServer(name)).toMatchObject({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 }));
|
||||
expect(JSON.stringify(listServerKeys())).not.toContain('test-key');
|
||||
expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: name })).toMatchObject({ ok: true, name });
|
||||
const spec = vi.mocked(komodoCreateDeployment).mock.calls.at(-1)![0];
|
||||
expect(spec.environment).toContainEqual({ variable: 'PLUTO_SERVER_KEY', value: 'test-key' });
|
||||
expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome' })).toMatchObject({ ok: false, kind: 'bad-request' });
|
||||
});
|
||||
|
||||
it('updates and removes keys independently of deployments and survives reopening the database', async () => {
|
||||
const { saveServerKey, getServerKey, removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
saveServerKey(serverName, ' first-key ');
|
||||
closeDb();
|
||||
expect(getServerKey(serverName)).toBe('first-key');
|
||||
saveServerKey(serverName, 'replacement');
|
||||
expect(getServerKey(serverName)).toBe('replacement');
|
||||
expect(removeServerKey(serverName)).toBe(true);
|
||||
expect(getServerKey(serverName)).toBeUndefined();
|
||||
expect(() => saveServerKey('t5_zombie_unknown_1', 'key')).toThrow();
|
||||
expect(() => saveServerKey(serverName, ' ')).toThrow();
|
||||
expect(() => saveServerKey(serverName, 'two words')).toThrow();
|
||||
});
|
||||
|
||||
it('fills the lowest gap and rejects stale previews before saving a key or deploying', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: serverName });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' });
|
||||
await destroyServer(serverName);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' }))
|
||||
.toMatchObject({ ok: false, kind: 'no-slot' });
|
||||
expect(getServerKey(serverName)).toBe('test-key');
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: serverName }))
|
||||
.toMatchObject({ ok: true, name: serverName });
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelling submitted creation and destruction', () => {
|
||||
it('retains a remotely created identity after cancellation without starting its container', async () => {
|
||||
const { queueCreateServer, processCreations, nextAvailablePort, nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = queueCreateServer(input); if (!queued.ok) throw new Error(queued.message);
|
||||
let release!: (value: {ok:true}) => void;
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;}));
|
||||
const work = processCreations();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true);
|
||||
expect(nextAvailablePort()).not.toBe(queued.port);
|
||||
expect(nextLobbyNumber('t5_zombie_cosmodrome')).toBe(2);
|
||||
release({ok:true}); await work;
|
||||
expect(getServer(queued.name)).toBeTruthy();
|
||||
expect(komodoDeploy).not.toHaveBeenCalled();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled');
|
||||
});
|
||||
|
||||
it('does not delete deployment metadata or files after cancelling container teardown', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
let release!: (value: {ok:true}) => void;
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;}));
|
||||
const queued=destroyServer(serverName); const work=processDestructions();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true);
|
||||
release({ok:true}); await work;
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(getServer(serverName)).toBeTruthy();
|
||||
expect(existsSync(join(fixture.t5,serverName))).toBe(true);
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled');
|
||||
});
|
||||
});
|
||||
@@ -1,403 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(),
|
||||
komodoDeploy: vi.fn(),
|
||||
komodoDestroyDeployment: vi.fn(),
|
||||
komodoDeleteDeployment: vi.fn(),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(),
|
||||
komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(),
|
||||
rconPlutoniumVersion: vi.fn(),
|
||||
rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'test-key' };
|
||||
function status(server: string, map = server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater') {
|
||||
return { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map, gametype: null, roster: [] } };
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('SERVER_PORT_MIN', '28960');
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodo.komodoCreateDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDeploy).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDeleteDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockImplementation(async () => {
|
||||
// Version evidence must be observed strictly after the launch timestamp.
|
||||
await new Promise(resolve => setTimeout(resolve, 2));
|
||||
return { revision: 5354, error: null };
|
||||
});
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server));
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
vi.mocked(kick).mockResolvedValue(undefined);
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.restoreAllMocks(); });
|
||||
|
||||
async function queueServers(count: number) {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
return Array.from({ length: count }, () => {
|
||||
const result = queueCreateServer(input);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
async function states() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'create' ORDER BY rowid").all() as { state: string }[]).map(row => row.state);
|
||||
}
|
||||
|
||||
describe('shared creation queue', () => {
|
||||
it('rechecks a lowered limit after awaiting a startup in the same pass', async () => {
|
||||
await queueServers(3);
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => {
|
||||
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
|
||||
return { ok: true };
|
||||
});
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
||||
});
|
||||
|
||||
it('applies live increases and decreases while preserving active startups', async () => {
|
||||
const jobs = await queueServers(4);
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoInspectHealthy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'still starting' });
|
||||
await processCreations();
|
||||
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']);
|
||||
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']);
|
||||
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[2].name
|
||||
? { ok: false, reason: 'still starting' } : { ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'done', 'awaiting-health', 'queued']);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'done', 'done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('starts only one server per pass and holds its slot until a later health check', async () => {
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
||||
await processCreations();
|
||||
expect(komodoInspectHealthy).toHaveBeenCalledWith(jobs[0].name);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
||||
expect(new Set(jobs.map(job => job.port)).size).toBe(3);
|
||||
});
|
||||
|
||||
it('keeps later boots queued through container, RCON, and expected-map failures', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await processCreations();
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' });
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON or fresh telemetry health: timeout' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon'));
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') });
|
||||
vi.mocked(komodoInspectHealthy).mockRejectedValueOnce(new Error('Temporary connection error'));
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('shares FIFO capacity across collections and direct callers, including equal timestamps', async () => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const collections = ['theater', 'moon'].map(mapSlug => {
|
||||
const blueprint = saveBlueprint({ name: mapSlug, entries: [{ game: 't5', mapSlug, count: 2 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
return queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name, 'test-key'])) });
|
||||
});
|
||||
db().prepare("UPDATE server_deployments SET started_at = '2026-01-01T00:00:00.000Z'").run();
|
||||
const direct = await createServer(input);
|
||||
if (!direct.ok) throw new Error(direct.message);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued', 'queued']);
|
||||
const second = getCollection(collections[1].collectionId!)!;
|
||||
expect(second.collection).toMatchObject({ queued: 2, provisioning: 0, verifying: 0 });
|
||||
expect(second.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: 't5_zombie_theater_1' }] });
|
||||
expect(second.servers.map(server => server.queuePosition)).toEqual([2, 3]);
|
||||
for (let pass = 0; pass < 5; pass++) await processCreations();
|
||||
expect(vi.mocked(komodoCreateDeployment).mock.calls.map(([spec]) => spec.name)).toEqual([
|
||||
't5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't5_zombie_moon_2', direct.name
|
||||
]);
|
||||
expect(await states()).toEqual(['done', 'done', 'done', 'done', 'done']);
|
||||
});
|
||||
|
||||
it('coalesces overlapping worker passes and direct calls while provisioning is pending', async () => {
|
||||
await queueServers(3);
|
||||
const { processCreations, createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const first = processCreations();
|
||||
await started;
|
||||
const second = processCreations();
|
||||
const direct = createServer(input);
|
||||
expect(await states()).toEqual(['running', 'queued', 'queued', 'queued']);
|
||||
release();
|
||||
await Promise.all([first, second, direct]);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued']);
|
||||
});
|
||||
|
||||
it('honors a higher limit without refilling its pass budget after immediate failures', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
await queueServers(5);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: false, kind: 'http-error', message: 'Host unavailable' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['failed', 'failed', 'queued', 'queued', 'queued']);
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: true });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(4);
|
||||
expect(await states()).toEqual(['failed', 'failed', 'awaiting-health', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('does not provision a selected job that was cancelled during an earlier startup', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const pass = processCreations();
|
||||
await started;
|
||||
finishDeployment(jobs[1].deploymentId, { ok: false, message: 'Cancelled before startup' });
|
||||
release();
|
||||
await pass;
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'failed', 'queued']);
|
||||
});
|
||||
|
||||
it('rechecks capacity when an existing server resumes verification during provisioning', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { finishDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const old = await createServer(input);
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
finishDeployment(old.deploymentId, { ok: false, message: 'Needs verification' });
|
||||
await queueServers(2);
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const pass = processCreations();
|
||||
await started;
|
||||
expect(resumeVerification(old.deploymentId)).toBe(true);
|
||||
release();
|
||||
await pass;
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('counts already verifying servers against a higher concurrency limit', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const jobs = await queueServers(4);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[0].name
|
||||
? { ok: false, reason: 'still booting' } : { ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(await states()).toEqual(['awaiting-health', 'done', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('retains durable reservations and the startup gate across an application restart', async () => {
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { processCreations: resumed, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
interruptDeployments();
|
||||
expect(await states()).toEqual(['interrupted', 'queued', 'queued']);
|
||||
expect(nextAvailablePort()).toBe(28963);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await resumed();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
await resumed();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('keeps an ambiguous startup response in verification without replaying its deployment', async () => {
|
||||
await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDeploy, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('Response lost after starting'));
|
||||
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
expect(log).toHaveBeenCalled();
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ok: false as const, kind: 'unreachable' as const, message: 'Request timed out after Komodo accepted it' },
|
||||
{ ok: false as const, kind: 'http-error' as const, httpStatus: 502, message: 'Gateway lost the startup response' }
|
||||
])('holds the slot and files for an uncertain structured response: $kind', async failure => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDeploy, komodoDeleteDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce(failure);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
expect(getServer(jobs[0].name)?.game_port).toBe(jobs[0].port);
|
||||
expect(existsSync(join(fixture.t5, jobs[0].name, 'configs/electric_cherry_map.cfg'))).toBe(true);
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('releases capacity on a confirmed deployment rejection', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDeploy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', httpStatus: 403, message: 'Access denied' });
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'queued']);
|
||||
expect(getServer(jobs[0].name)).toBeUndefined();
|
||||
expect(existsSync(join(fixture.t5, jobs[0].name))).toBe(false);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('retains an interrupted startup slot while a newer destruction is still waiting', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
expect(destroyServer(jobs[0].name)).toMatchObject({ ok: true });
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
||||
interruptDeployments();
|
||||
expect(await states()).toEqual(['interrupted', 'queued']);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('releases a startup slot after its server is removed', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await processCreations();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['cancelled', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('never verifies a replacement registered under the same server name', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(28980, jobs[0].name);
|
||||
await processCreations();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(jobs[0].name);
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('creation concurrency configuration', () => {
|
||||
it('defaults to one startup', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined);
|
||||
const { fleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
expect(fleetSettings().createConcurrency).toBe(1);
|
||||
});
|
||||
|
||||
it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value);
|
||||
const { fleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
expect(() => fleetSettings()).toThrow(/Concurrent server startups/);
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,7 @@ const path = (server: string) => join(fixture.t5, server, 'runtime/plutonium/sto
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(names);
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', '');
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import type { CollectionPreview } from '../../src/lib/deployment-plan.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeploy: vi.fn(async () => ({ ok: true })),
|
||||
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => {
|
||||
@@ -34,14 +26,10 @@ const entries = [
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
await prepareNativeHost();
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'private-api-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', 'private-rcon-password');
|
||||
vi.stubEnv('SERVER_PORT_MIN', '28960');
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
@@ -67,26 +55,6 @@ describe('multi-server blueprints', () => {
|
||||
expect(getCollection(result.collectionId!)?.queue.concurrency).toBe(3);
|
||||
});
|
||||
|
||||
it('saves a mixed-game lineup and previews unique slots and global ports without reserving them', async () => {
|
||||
const { blueprint, preview } = await previewLineup();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
|
||||
expect(blueprint.totalServers).toBe(5);
|
||||
expect(blueprint.entries.map(entry => entry.count)).toEqual([2, 1, 2]);
|
||||
expect(blueprint).not.toHaveProperty('recipe');
|
||||
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string };
|
||||
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries });
|
||||
expect(preview.ready).toBe(true);
|
||||
expect(preview.launchConcurrency).toBe(1);
|
||||
expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']);
|
||||
expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]);
|
||||
expect(JSON.stringify(preview)).not.toContain('private-');
|
||||
expect(preview.entries[0].recipe.files[0]).not.toHaveProperty('content');
|
||||
expect(() => previewCreation({ blueprintId: blueprint.id })).toThrow(/multiple servers/);
|
||||
});
|
||||
|
||||
it('reads a legacy one-server blueprint as a map selection and saves edits without snapshots', async () => {
|
||||
const { captureRecipe, listBlueprints, updateBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
@@ -112,149 +80,15 @@ describe('multi-server blueprints', () => {
|
||||
expect(listBlueprints()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('uses current promotions for legacy collection blueprints while retaining counts and exact T6 variants', async () => {
|
||||
const { captureRecipe, listBlueprints } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const artifact = createScript({ name: 'Legacy telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id);
|
||||
const legacyEntries = entries.map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipe(game as 't5' | 't6', mapSlug) }));
|
||||
db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy-lineup', 'Legacy lineup', new Date().toISOString(), JSON.stringify({ format: 2, entries: legacyEntries }));
|
||||
const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id);
|
||||
closeDb();
|
||||
expect(listBlueprints()[0].entries.map(({ mapName: _mapName, ...entry }) => entry)).toEqual(entries);
|
||||
const preview = previewCollection('legacy-lineup');
|
||||
expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id }));
|
||||
expect(preview.entries[2].recipe.mapSlug).toBe('town');
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 });
|
||||
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
|
||||
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry');
|
||||
}
|
||||
expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
|
||||
});
|
||||
|
||||
it.each([[], [{ game: 't5', mapSlug: 'theater', count: 0 }], [{ game: 't5', mapSlug: 'theater', count: 21 }],
|
||||
[{ game: 't5', mapSlug: 'theater', count: 1.5 }], [{ game: 't5', mapSlug: 'theater', count: '2' }],
|
||||
[{ game: 't5', mapSlug: 'town', count: 1 }], [entries[0], entries[0]]].map(input => ({ input })))('rejects an invalid lineup: $input', async ({ input }) => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
expect(() => saveBlueprint({ name: 'Invalid', entries: input })).toThrow();
|
||||
});
|
||||
|
||||
it('reports insufficient capacity without preventing blueprint creation', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28961');
|
||||
const { blueprint, preview } = await previewLineup();
|
||||
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(blueprint.totalServers).toBe(5);
|
||||
expect(preview.ready).toBe(false);
|
||||
expect(preview.servers.filter(server => server.port === null)).toHaveLength(3);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/);
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('atomic collection launches', () => {
|
||||
it('fills the target shortfall and counts registered/in-flight identities only once', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (let i = 0; i < 2; i++) expect((await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' })).ok).toBe(true);
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const blueprint = saveBlueprint({ name: 'Friday targets', entries: [
|
||||
{ game: 't5', mapSlug: 'theater', count: 4 }, { game: 't5', mapSlug: 'factory', count: 2 }
|
||||
] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.entries.map(({ count, existing, missing }) => ({ count, existing, missing }))).toEqual([
|
||||
{ count: 4, existing: 2, missing: 2 }, { count: 2, existing: 0, missing: 2 }
|
||||
]);
|
||||
expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_3', 't5_zombie_theater_4', 't5_zombie_factory_1', 't5_zombie_factory_2']);
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
expect(getCollection(queued.collectionId!)?.collection.total).toBe(4);
|
||||
expect(previewCollection(blueprint.id).servers).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps excess and unrelated servers, including stopped servers, without requiring launch settings for a no-op', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE servers SET container_state = 'stopped'").run();
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
// Fixture-only configuration changes simulate missing integrations after creation.
|
||||
Object.assign(config.rcon, { password: '' });
|
||||
Object.assign(config.komodo, { targetServer: '' });
|
||||
const blueprint = saveBlueprint({ name: 'One Kino', entries: [{ game: 't5', mapSlug: 'theater', count: 1 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.entries[0]).toMatchObject({ existing: 2, missing: 0 });
|
||||
expect(preview).toMatchObject({ ready: true, blockers: [], servers: [] });
|
||||
expect(queueCollection({ previewId: preview.id })).toEqual({ ok: true, collectionId: null });
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM servers').all()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('rechecks shrinking deficits at submission and rejects an older additive preview', async () => {
|
||||
const { preview, blueprint } = await previewLineup();
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { queueCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' });
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/changed/);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
|
||||
const fresh = previewCollection(blueprint.id);
|
||||
expect(fresh.servers).toHaveLength(4);
|
||||
const extra = [...fresh.servers, { ...fresh.servers[0], name: 't5_zombie_theater_20', port: 28980 }];
|
||||
db().prepare('UPDATE server_collection_previews SET servers = ? WHERE id = ?').run(JSON.stringify(extra), fresh.id);
|
||||
expect(() => queueCollection({ previewId: fresh.id, keys: keysFor(fresh) })).toThrow(/changed/);
|
||||
});
|
||||
|
||||
it('matches exact T6 variants and excludes failed creations and pending destructions', async () => {
|
||||
const { createServer, queueCreateServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const farm = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm-key' });
|
||||
const town = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town-key' });
|
||||
if (!farm.ok || !town.ok) throw new Error('Fixture creation failed');
|
||||
destroyServer(town.name);
|
||||
const failed = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'failed-key' });
|
||||
if (!failed.ok) throw new Error(failed.message);
|
||||
finishDeployment(failed.deploymentId, { ok: false, message: 'Fixture failure' });
|
||||
const blueprint = saveBlueprint({ name: 'Town target', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.entries[0]).toMatchObject({ existing: 0, missing: 1 });
|
||||
expect(preview.servers[0].name).toBe('t6_zombie_town_2');
|
||||
});
|
||||
|
||||
it('queues all five servers, persists one group, and completes each independently', async () => {
|
||||
const { preview } = await previewLineup();
|
||||
const { queueCollection, getCollection, listCollections } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 });
|
||||
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]);
|
||||
closeDb();
|
||||
await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 });
|
||||
expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] });
|
||||
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]);
|
||||
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
|
||||
const detail = getCollection(queued.collectionId!)!;
|
||||
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 });
|
||||
expect(listCollections()).toHaveLength(1);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(5);
|
||||
expect(getDeployment(detail.servers[0].id)?.collection).toEqual({ id: queued.collectionId, name: 'Friday Zombies' });
|
||||
expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
|
||||
expect(JSON.stringify(detail)).not.toContain('key-t5');
|
||||
});
|
||||
|
||||
it('rolls back all earlier jobs and key changes if the final server has no valid key', async () => {
|
||||
const { preview } = await previewLineup();
|
||||
@@ -264,7 +98,7 @@ describe('atomic collection launches', () => {
|
||||
saveServerKey(preview.servers[0].name!, 'original-key');
|
||||
const keys = keysFor(preview);
|
||||
delete keys[preview.servers[4].name!];
|
||||
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/No saved key/);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/Save a Plutonium server key/);
|
||||
for (const table of ['server_launch_collections', 'server_collection_members', 'server_creation_jobs', 'server_deployments', 'server_launch_previews', 'server_launch_recipes']) {
|
||||
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
}
|
||||
@@ -276,18 +110,6 @@ describe('atomic collection launches', () => {
|
||||
expect(queueCollection({ previewId: preview.id, keys: keysFor(preview) }).ok).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects changed capacity before saving keys or partially queueing the lineup', async () => {
|
||||
const { preview } = await previewLineup();
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
queueCreateServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', plutoKey: 'other-server-key' });
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/availability changed/);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
|
||||
expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1);
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('reuses saved keys and returns the same collection on resubmission after restart and expiry', async () => {
|
||||
const { preview, blueprint } = await previewLineup();
|
||||
const { queueCollection, previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
@@ -309,52 +131,6 @@ describe('atomic collection launches', () => {
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps the reviewed lineup after blueprint edits and deletion while using current promotions', async () => {
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { script, version } = createScript({ name: 'Classic perks', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'perks.gsc', content: Buffer.from('original') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
const { preview, blueprint } = await previewLineup();
|
||||
const { updateBlueprint, deleteBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const newer = uploadNewVersion(script.id, Buffer.from('new version'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newer.id, script.id);
|
||||
updateBlueprint(blueprint.id, { name: 'Changed', entries: [{ ...entries[0], count: 1 }] });
|
||||
deleteBlueprint(blueprint.id);
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const queuedDetail = getCollection(queued.collectionId!)!;
|
||||
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
for (const server of queuedDetail.servers.filter(server => server.server.startsWith('t5_'))) {
|
||||
expect(recipeForDeployment(server.id)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newer.id }));
|
||||
}
|
||||
const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id);
|
||||
uploadNewVersion(script.id, Buffer.from('unpromoted draft'));
|
||||
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 });
|
||||
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
|
||||
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued');
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps a provisioning failure separate while the other servers complete', async () => {
|
||||
const { preview } = await previewLineup();
|
||||
const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' });
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 });
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
|
||||
const detail = getCollection(queued.collectionId!)!;
|
||||
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 });
|
||||
expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') });
|
||||
expect(detail.servers.slice(1).every(server => server.state === 'done')).toBe(true);
|
||||
});
|
||||
|
||||
it('validates collection API payloads and rejects an unused expired preview', async () => {
|
||||
const { preview, blueprint } = await previewLineup();
|
||||
const { POST: singlePreview } = await import('../../src/routes/api/deployments/preview/+server.js');
|
||||
@@ -382,52 +158,6 @@ describe('matching a blueprint exactly', () => {
|
||||
return saveBlueprint({ name: 'Exact lineup', entries });
|
||||
}
|
||||
|
||||
it('keeps the lowest deployed lobbies, removes excess and unrelated maps, and adds only the shortfall', async () => {
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (const mapSlug of ['theater', 'theater', 'moon']) await createServer({ gameMode: 'zombie', mapSlug, plutoKey: 'existing' });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const before = db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get();
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }, { game: 't6', mapSlug: 'town', count: 1 }]);
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.retainedServers).toEqual(['t5_zombie_theater_1']);
|
||||
expect(preview.removals.map(server => server.name)).toEqual(['t5_zombie_moon_1', 't5_zombie_theater_2']);
|
||||
expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_1']);
|
||||
const { komodoDestroyDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ mode: 'exact', total: 3, active: 3, queued: 3 });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
await processCreations();
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 3, done: 3, active: 0 });
|
||||
expect(getCollection(result.collectionId!)?.servers.filter(server => server.action === 'destroy')).toHaveLength(2);
|
||||
expect(db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get()).toEqual(before);
|
||||
expect(db().prepare('SELECT name FROM servers ORDER BY name').all()).toEqual([{ name: 't5_zombie_theater_1' }, { name: 't6_zombie_town_1' }]);
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalledWith('t5_zombie_theater_1');
|
||||
});
|
||||
|
||||
it('supports removal-only plans, shares existing removals, and records retained servers', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const first = previewCollection(plan.id, 'exact');
|
||||
expect(first.servers).toEqual([]);
|
||||
const a = queueCollection({ previewId: first.id });
|
||||
const b = queueCollection({ previewId: previewCollection(plan.id, 'exact').id });
|
||||
const firstDetail = getCollection(a.collectionId!)!;
|
||||
expect(firstDetail.retainedServers).toEqual(['t5_zombie_theater_1']);
|
||||
expect(firstDetail.servers).toHaveLength(1);
|
||||
expect(getCollection(b.collectionId!)?.servers[0].id).toBe(firstDetail.servers[0].id);
|
||||
expect(firstDetail.servers[0]).toMatchObject({ action: 'destroy', state: 'waiting', queuePosition: null });
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
expect(getDeployment(firstDetail.servers[0].id)?.collection).toBeTruthy();
|
||||
});
|
||||
|
||||
it('cancels unstarted excess creations without booting them and retains their saved keys', async () => {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const first = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'first' });
|
||||
@@ -442,21 +172,6 @@ describe('matching a blueprint exactly', () => {
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 });
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(second.name)).toBe('second');
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects fleet changes even when the create shortfall stays the same', async () => {
|
||||
const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'new' });
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow(/fleet changed/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('rolls back removals and creation cancellations when a required key is missing', async () => {
|
||||
@@ -466,128 +181,13 @@ describe('matching a blueprint exactly', () => {
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow(/No saved key/);
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow(/Save a Plutonium server key/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
|
||||
for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('reuses a full fleet port only after empty-server removal, across restart and repeated submission', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview).toMatchObject({ ready: true, servers: [{ port: 28960 }] });
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
expect(queueCollection({ previewId: preview.id })).toEqual(result);
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
for (const players of [2, null]) {
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('Waiting for removal');
|
||||
}
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 2, done: 2, active: 0 });
|
||||
});
|
||||
|
||||
it('does not reuse a port after a removal is cancelled', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
expect(cancelRemoval(getCollection(result.collectionId!)!.servers.find(server => server.action === 'destroy')!.id)).toBe(true);
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.collection.failed).toBe(1);
|
||||
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('(cancelled)');
|
||||
});
|
||||
|
||||
it('allows independent launches to pass a failed removal without exceeding the startup limit', async () => {
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Could not stop container' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
const creates = getCollection(result.collectionId!)!.servers.filter(server => server.action === 'create');
|
||||
expect(creates[0]).toMatchObject({ state: 'queued', stage: expect.stringContaining('(failed)') });
|
||||
expect(creates[1]).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getCollection(result.collectionId!)!.collection).toMatchObject({ failed: 1, queued: 1, verifying: 1 });
|
||||
});
|
||||
|
||||
it('rejects insufficient final capacity before cancelling any existing creation', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.ready).toBe(false);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('requires a new review when a pending removal is cancelled', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const old = await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
const removal = destroyServer(old.name);
|
||||
if (!removal.ok) throw new Error(removal.message);
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
cancelRemoval(removal.deploymentId!);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/fleet changed/);
|
||||
expect(previewCollection(plan.id, 'exact')).toMatchObject({ servers: [], removals: [], retainedServers: [old.name] });
|
||||
});
|
||||
|
||||
it('keeps T6 variants distinct and leaves a matching fleet untouched', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town' });
|
||||
const plan = await blueprint([{ game: 't6', mapSlug: 'town', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const noOp = previewCollection(plan.id, 'exact');
|
||||
expect(noOp).toMatchObject({ retainedServers: ['t6_zombie_town_1'], removals: [], servers: [] });
|
||||
expect(queueCollection({ previewId: noOp.id })).toEqual({ ok: true, collectionId: null });
|
||||
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm' });
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.retainedServers).toEqual(['t6_zombie_town_1']);
|
||||
expect(preview.removals.map(server => server.name)).toEqual(['t6_zombie_farm_1']);
|
||||
});
|
||||
|
||||
it('defaults old previews and collections to fill and validates the preview mode API', async () => {
|
||||
const { blueprint: plan } = await previewLineup();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(),
|
||||
komodoDeploy: vi.fn(),
|
||||
komodoDestroyDeployment: vi.fn(),
|
||||
komodoDeleteDeployment: vi.fn(),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(),
|
||||
komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void;
|
||||
let reject!: (error: Error) => void;
|
||||
const promise = new Promise<void>((done, fail) => { resolve = done; reject = fail; });
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function command() {
|
||||
return { started: deferred(), response: deferred() };
|
||||
}
|
||||
|
||||
const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`);
|
||||
let fixture: Fixture;
|
||||
let commands: Map<string, { destroy: ReturnType<typeof command>; delete: ReturnType<typeof command> }>;
|
||||
let passes: Promise<void>[];
|
||||
let runningAtSubmission: number[];
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture(servers);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
|
||||
KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value);
|
||||
commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }]));
|
||||
passes = [];
|
||||
runningAtSubmission = [];
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const [index, server] of servers.entries()) {
|
||||
db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`)
|
||||
.run(server, server, 28960 + index, join(fixture.t5, server));
|
||||
}
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementation(async server => {
|
||||
runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'")
|
||||
.get() as { n: number }).n);
|
||||
const gate = commands.get(server)!.destroy;
|
||||
gate.started.resolve();
|
||||
await gate.response.promise;
|
||||
return { ok: true };
|
||||
});
|
||||
vi.mocked(komodoDeleteDeployment).mockImplementation(async server => {
|
||||
const gate = commands.get(server)!.delete;
|
||||
gate.started.resolve();
|
||||
await gate.response.promise;
|
||||
return { ok: true };
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const server of servers) complete(server);
|
||||
await Promise.allSettled(passes);
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function complete(server: string) {
|
||||
commands.get(server)!.destroy.response.resolve();
|
||||
commands.get(server)!.delete.response.resolve();
|
||||
}
|
||||
|
||||
function track(pass: Promise<void>) {
|
||||
passes.push(pass);
|
||||
return pass;
|
||||
}
|
||||
|
||||
async function queue(count = servers.length) {
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
return servers.slice(0, count).map(server => {
|
||||
const result = destroyServer(server);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
return result.deploymentId!;
|
||||
});
|
||||
}
|
||||
|
||||
async function states() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid")
|
||||
.all() as { state: string }[]).map(row => row.state);
|
||||
}
|
||||
|
||||
describe('destruction concurrency', () => {
|
||||
it('defaults to one full removal at a time and refills the slot in the same pass', async () => {
|
||||
await queue(2);
|
||||
const { fleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(fleetSettings().destroyConcurrency).toBe(1);
|
||||
const pass = track(processDestructions());
|
||||
await commands.get(servers[0])!.destroy.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['running', 'waiting']);
|
||||
complete(servers[0]);
|
||||
await commands.get(servers[1])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['done', 'running']);
|
||||
complete(servers[1]);
|
||||
await pass;
|
||||
expect(await states()).toEqual(['done', 'done']);
|
||||
expect(runningAtSubmission).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
const overlapping = track(processDestructions());
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']);
|
||||
commands.get(servers[0])!.destroy.response.resolve();
|
||||
await commands.get(servers[0])!.delete.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
commands.get(servers[0])!.delete.response.resolve();
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['done', 'running', 'running', 'waiting']);
|
||||
complete(servers[1]);
|
||||
await commands.get(servers[3])!.destroy.started.promise;
|
||||
complete(servers[2]); complete(servers[3]);
|
||||
await Promise.all([pass, overlapping]);
|
||||
expect(await states()).toEqual(['done', 'done', 'done', 'done']);
|
||||
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4);
|
||||
expect(Math.max(...runningAtSubmission)).toBe(2);
|
||||
});
|
||||
|
||||
it('lets active removals finish after lowering the limit and applies it to new admissions', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 });
|
||||
complete(servers[0]);
|
||||
await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting']));
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
complete(servers[1]);
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(await states()).toEqual(['done', 'done', 'running', 'waiting']);
|
||||
complete(servers[2]);
|
||||
await commands.get(servers[3])!.destroy.started.promise;
|
||||
complete(servers[3]);
|
||||
await pass;
|
||||
expect(runningAtSubmission.slice(2)).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => {
|
||||
await queue(3);
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
if (failure === 'rejected') {
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => {
|
||||
commands.get(servers[0])!.destroy.started.resolve();
|
||||
await commands.get(servers[0])!.destroy.response.promise;
|
||||
return { ok: false, kind: 'http-error', message: 'Teardown rejected' };
|
||||
});
|
||||
}
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost'));
|
||||
else commands.get(servers[0])!.destroy.response.resolve();
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['failed', 'running', 'running']);
|
||||
complete(servers[1]); complete(servers[2]);
|
||||
await pass;
|
||||
expect(await states()).toEqual(['failed', 'done', 'done']);
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]);
|
||||
});
|
||||
|
||||
it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null);
|
||||
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1]
|
||||
? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' });
|
||||
for (const server of servers) complete(server);
|
||||
await track(processDestructions());
|
||||
expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']);
|
||||
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2));
|
||||
expect(rconGetPlayerCount).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
@@ -31,7 +31,7 @@ async function applyDriftDeployment(input: import('../../src/lib/server/fleet/dr
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(['t5_zombie_theater_1']);
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
// Ensure the three per-kind destinations exist under the fixture tree.
|
||||
await mkdir(join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/storage/t5/raw/scripts/sp'), { recursive: true });
|
||||
|
||||
@@ -37,7 +37,7 @@ async function setup() {
|
||||
describe('fleet event refreshes', () => {
|
||||
it('refreshes server history through one scoped request without replacing the fleet snapshot', async () => {
|
||||
const store = await import('../../src/lib/ui/stores/fleet.js');
|
||||
const integrations = { rcon: true, komodo: true, rconTimeoutMs: 2000, defaultRconHost: '' };
|
||||
const integrations = { rcon: true, rconTimeoutMs: 2000, defaultRconHost: '' };
|
||||
const snapshot = { servers: [], operations: [], integrations, serverNow: Date.now() };
|
||||
store.hydrateFleet(snapshot);
|
||||
const originalFleet = get(store.fleet);
|
||||
|
||||
@@ -54,7 +54,6 @@ describe('Fleet discovery and match freshness', () => {
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('persisted Fleet operations', () => {
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
|
||||
@@ -75,7 +74,7 @@ describe('persisted Fleet operations', () => {
|
||||
it('recovers waiting and recovering campaigns without needing a live event', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
||||
handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('one', 'promote', ?)").run(iso(-2 * 86_400_000));
|
||||
handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players) VALUES ('one', 'server', 1, 'pending', 4)").run();
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
@@ -109,7 +108,7 @@ describe('persisted Fleet operations', () => {
|
||||
it('surfaces orphaned recovery errors and beta gates', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
||||
handle.prepare("INSERT INTO restart_rollout (server, requested_at, error) VALUES ('server', ?, 'RCON not answering')").run(iso());
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now)[0]).toMatchObject({ kind: 'recovery', state: 'attention', detail: 'RCON not answering' });
|
||||
|
||||
@@ -10,11 +10,11 @@ beforeEach(async () => {
|
||||
fixture = await newFixture([]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().transaction(() => {
|
||||
const server = db().prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)');
|
||||
const server = db().prepare('INSERT INTO servers(name) VALUES (?)');
|
||||
const operation = db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by)
|
||||
VALUES(?,?,'restart',?,'Complete',?,?,'test')`);
|
||||
for (const name of ['one', 'two']) {
|
||||
server.run(name, name);
|
||||
server.run(name);
|
||||
for (let i = 0; i < 100; i++) operation.run(`${name}-${i}`, name, 'done', iso(-i), iso(-i));
|
||||
operation.run(`${name}-old`, name, 'done', iso(-86_400_001), iso(-86_400_001));
|
||||
operation.run(`${name}-failed`, name, 'failed', iso(), iso());
|
||||
|
||||
@@ -16,7 +16,7 @@ let fetchMock: ReturnType<typeof vi.fn>;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture();
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
fetchMock = vi.fn(async (input: string | URL) => String(input).startsWith('https://api.github.com/')
|
||||
? Response.json(metadata()) : new Response(content));
|
||||
|
||||
@@ -2,7 +2,6 @@ import { randomUUID } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() }));
|
||||
|
||||
const server = 'managed';
|
||||
@@ -22,11 +21,12 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreA
|
||||
async function setup() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const database = db();
|
||||
database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)')
|
||||
.run(server, server, new Date(now).toISOString());
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
database.prepare('INSERT INTO servers(name,created_at,game_port) VALUES(?,?,28960)')
|
||||
.run(server, new Date(now).toISOString());
|
||||
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy);
|
||||
const inspect = vi.mocked(nativeHealth).mockResolvedValue(healthy);
|
||||
const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing);
|
||||
return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') };
|
||||
}
|
||||
@@ -228,37 +228,6 @@ describe('shared monitoring observations', () => {
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => {
|
||||
const api = await setup();
|
||||
await api.inspectForMonitoring(server);
|
||||
api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server);
|
||||
await api.inspectForMonitoring(server);
|
||||
expect(api.inspect).toHaveBeenCalledTimes(2);
|
||||
api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server);
|
||||
api.rcon.mockImplementationOnce(async () => {
|
||||
api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server);
|
||||
return playing;
|
||||
});
|
||||
expect((await api.rconForMonitoring(server)).result).toEqual(playing);
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(1);
|
||||
api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server);
|
||||
await api.rconForMonitoring(server);
|
||||
expect(api.rcon).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => {
|
||||
const api = await setup();
|
||||
const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring;
|
||||
const request = kind === 'inspect' ? api.inspect : api.rcon;
|
||||
const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at';
|
||||
await call(server);
|
||||
vi.setSystemTime(now + 1000);
|
||||
api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server);
|
||||
await call(server);
|
||||
expect(request).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => {
|
||||
const api = await setup();
|
||||
const { instanceId } = await native(api);
|
||||
|
||||
@@ -16,10 +16,10 @@ describe('host detail instances', () => {
|
||||
db().prepare(`INSERT INTO native_instances (id, server, host_id, generation, library_id, image, port, created_at, retired_at)
|
||||
VALUES (?, ?, ?, 1, 'library', 'runtime:test', 28960, ?, ?)`)
|
||||
.run(name, name, hostId, now, retired ? now : null);
|
||||
db().prepare(`INSERT INTO servers (name, komodo_name,
|
||||
db().prepare(`INSERT INTO servers (name,
|
||||
container_state, container_checked_at, last_seen_at, last_players, destroyed_at)
|
||||
VALUES (?, ?, 'running', ?, ?, 2, ?)`)
|
||||
.run(name, name, now, now, destroyed ? now : null);
|
||||
VALUES (?, 'running', ?, ?, 2, ?)`)
|
||||
.run(name, now, now, destroyed ? now : null);
|
||||
};
|
||||
add('running', host.id);
|
||||
add('unavailable', host.id);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() }));
|
||||
let fixture: Fixture;
|
||||
let at: number;
|
||||
beforeEach(async () => {
|
||||
@@ -15,8 +14,9 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
|
||||
async function setup(count = 5, hostCount = 1) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
vi.mocked(nativeHealth).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const hostIds: string[] = [];
|
||||
for (let h = 0; h < hostCount; h++) {
|
||||
const host = hosts.createHost({ name: `Host ${h}` }); hostIds.push(host.id);
|
||||
@@ -26,7 +26,7 @@ async function setup(count = 5, hostCount = 1) {
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
||||
for (let i = 0; i < count; i++) {
|
||||
const name = `server-${h}-${i}`;
|
||||
db().prepare('INSERT INTO servers(name,komodo_name,created_at) VALUES(?,?,?)').run(name, '', new Date(at - 1000).toISOString());
|
||||
db().prepare('INSERT INTO servers(name,created_at) VALUES(?,?)').run(name, new Date(at - 1000).toISOString());
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES(?,?,?,'library','image',?,?)`).run(`instance-${h}-${i}`, name, host.id, 28960 + i, new Date(at - 1000).toISOString());
|
||||
}
|
||||
@@ -41,7 +41,7 @@ async function setup(count = 5, hostCount = 1) {
|
||||
? { ok: true, results: targets.map(target => ({ ...target, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } })) }
|
||||
: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } });
|
||||
});
|
||||
return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(komodoInspectHealthy), hostIds, operations, complete };
|
||||
return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(nativeHealth), hostIds, operations, complete };
|
||||
}
|
||||
|
||||
describe('host status observations', () => {
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { komodoInspectHealthy } from '../../src/lib/server/integrations/komodo.js';
|
||||
|
||||
vi.mock('../../src/lib/server/config.js', () => ({
|
||||
config: {
|
||||
komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' },
|
||||
httpTimeoutMs: 1000
|
||||
},
|
||||
komodoConfigured: () => true
|
||||
}));
|
||||
vi.mock('../../src/lib/server/fleet/servers.js', () => ({
|
||||
getServer: () => ({ komodo_type: 'deployment', komodo_name: 'saved-resource' })
|
||||
}));
|
||||
|
||||
const fetchMock = vi.fn<typeof fetch>();
|
||||
|
||||
beforeEach(() => {
|
||||
fetchMock.mockReset();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('Komodo health endpoint fallback', () => {
|
||||
it('limits an unreachable probe to one timeout and preserves its diagnosis', async () => {
|
||||
vi.useFakeTimers();
|
||||
fetchMock.mockImplementation((_url, options) => new Promise((_resolve, reject) => {
|
||||
options?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
|
||||
}));
|
||||
|
||||
const result = komodoInspectHealthy('server');
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(await result).toMatchObject({
|
||||
ok: false,
|
||||
reason: 'timed out after 1000ms dialing http://komodo.test/read/GetDeploymentContainer'
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves the network cause without probing alternate endpoints', async () => {
|
||||
fetchMock.mockRejectedValue(new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' })
|
||||
}));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringContaining('ECONNREFUSED: connection refused')
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([400, 401, 403, 429, 500, 502, 503])('stops on HTTP %i without hiding the response', async status => {
|
||||
fetchMock.mockResolvedValue(new Response('request failed', { status }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({
|
||||
ok: false, reason: `HTTP ${status}`, snippet: 'request failed'
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[404, 'Not Found'],
|
||||
[400, 'unknown variant `GetDeploymentContainer`'],
|
||||
[405, 'unsupported method'],
|
||||
[500, 'unknown method'],
|
||||
[501, 'method not found']
|
||||
])('tries a compatible endpoint after HTTP %i: %s', async (status, message) => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(new Response(message as string, { status: status as number }))
|
||||
.mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
|
||||
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
|
||||
'http://komodo.test/read/GetDeploymentContainer',
|
||||
'http://komodo.test/read/InspectDeploymentContainer'
|
||||
]);
|
||||
});
|
||||
|
||||
it('continues past successful responses with an unknown shape', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json({ config: { image: 'server' } }))
|
||||
.mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not overwrite an unhealthy container state with a later response', async () => {
|
||||
fetchMock.mockResolvedValueOnce(Response.json({ State: { Status: 'exited' } }));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({ ok: false, state: 'exited', reason: 'container state: exited' });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves malformed response diagnostics', async () => {
|
||||
fetchMock.mockResolvedValueOnce(new Response('not json'));
|
||||
|
||||
expect(await komodoInspectHealthy('server')).toMatchObject({
|
||||
ok: false, reason: expect.stringContaining('unparseable json:'), snippet: 'not json'
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,36 +0,0 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn() }));
|
||||
beforeEach(() => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: true })));
|
||||
});
|
||||
afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); });
|
||||
|
||||
it.each(['start', 'stop'] as const)('sends %s to the saved resource for each Komodo type', async action => {
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const verb = action === 'start' ? 'Start' : 'Stop';
|
||||
for (const [type, suffix, body] of [
|
||||
['deployment', 'Deployment', { deployment: 'saved-resource' }],
|
||||
['stack', 'Stack', { stack: 'saved-resource' }],
|
||||
['container', 'Container', { server: 'host', container: 'saved-resource' }]
|
||||
] as const) {
|
||||
vi.mocked(getServer).mockReturnValue({ komodo_type: type, komodo_name: 'saved-resource', komodo_server: 'host' } as ReturnType<typeof getServer>);
|
||||
expect(await komodoRuntimeAction('ec-server-id', action)).toEqual({ ok: true });
|
||||
expect(fetch).toHaveBeenLastCalledWith(`http://komodo.test/execute/${verb}${suffix}`, expect.objectContaining({ method: 'POST', body: JSON.stringify(body) }));
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects missing resource identity and surfaces a rejected stop response', async () => {
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(getServer).mockReturnValue(undefined);
|
||||
expect(await komodoRuntimeAction('missing', 'stop')).toMatchObject({ ok: false, kind: 'no-name' });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
vi.mocked(getServer).mockReturnValue({ komodo_type: 'deployment', komodo_name: 'saved' } as ReturnType<typeof getServer>);
|
||||
vi.mocked(fetch).mockResolvedValueOnce(Response.json({ success: false, message: 'rejected' }));
|
||||
expect(await komodoRuntimeAction('server', 'stop')).toMatchObject({ ok: false, message: 'rejected' });
|
||||
});
|
||||
@@ -1,88 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { komodoEnsureTag, komodoUpdateTagsOnResource } from '../../src/lib/server/integrations/komodo.js';
|
||||
|
||||
vi.mock('../../src/lib/server/config.js', () => ({
|
||||
config: {
|
||||
komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' },
|
||||
httpTimeoutMs: 1000
|
||||
},
|
||||
komodoConfigured: () => true
|
||||
}));
|
||||
|
||||
const fetchMock = vi.fn<typeof fetch>();
|
||||
const gameId = '6aa1fb6bc27e735cc0767f5a';
|
||||
const mapId = '6aabe8567a227ba7d9aad3e9';
|
||||
const tags = [
|
||||
{ _id: { $oid: gameId }, name: 't5', owner: '6aa1e0ba91f87b7712e8caf8', color: 'Slate' },
|
||||
{ _id: { $oid: mapId }, name: 'zombie_cod5_sumpf', owner: '6aa1e0ba91f87b7712e8caf8', color: 'Slate' }
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
fetchMock.mockReset();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('Komodo tags', () => {
|
||||
it('reuses the tags from the logged Mongo response and sends string IDs to the deployment', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json(tags))
|
||||
.mockResolvedValueOnce(Response.json(tags))
|
||||
.mockResolvedValueOnce(Response.json({}));
|
||||
|
||||
const game = await komodoEnsureTag('t5');
|
||||
const map = await komodoEnsureTag('zombie_cod5_sumpf');
|
||||
expect(game).toEqual({ ok: true, id: gameId });
|
||||
expect(map).toEqual({ ok: true, id: mapId });
|
||||
if (!game.ok || !map.ok) throw new Error('Expected both tags to resolve');
|
||||
expect(await komodoUpdateTagsOnResource('Deployment', 't5-sumpf-1', [game.id, map.id]))
|
||||
.toEqual({ ok: true });
|
||||
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
|
||||
'http://komodo.test/read/ListTags',
|
||||
'http://komodo.test/read/ListTags',
|
||||
'http://komodo.test/write/UpdateResourceMeta'
|
||||
]);
|
||||
expect(JSON.parse(fetchMock.mock.calls[2][1]!.body as string)).toEqual({
|
||||
target: { type: 'Deployment', id: 't5-sumpf-1' }, tags: [gameId, mapId]
|
||||
});
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([{ id: gameId }, { _id: gameId }])('preserves string ID support: %j', async (id) => {
|
||||
fetchMock.mockResolvedValueOnce(Response.json([{ ...id, name: 't5' }]));
|
||||
expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reads an Extended JSON ID from CreateTag without another lookup', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json([]))
|
||||
.mockResolvedValueOnce(Response.json([]))
|
||||
.mockResolvedValueOnce(Response.json(tags[0]));
|
||||
expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
expect(fetchMock.mock.calls[2][0]).toBe('http://komodo.test/write/CreateTag');
|
||||
});
|
||||
|
||||
it('recovers when another caller creates the tag after the initial lookup', async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json([]))
|
||||
.mockResolvedValueOnce(Response.json([]))
|
||||
.mockResolvedValueOnce(Response.json({ error: 'E11000 duplicate key error' }, { status: 500 }))
|
||||
.mockResolvedValueOnce(Response.json(tags));
|
||||
expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId });
|
||||
expect(fetchMock).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it.each([null, {}, { $oid: '' }, { $oid: 123 }])('rejects malformed Mongo IDs: %j', async (_id) => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(Response.json([{ name: 't5', _id }]))
|
||||
.mockResolvedValueOnce(Response.json({ error: 'create failed' }, { status: 400 }));
|
||||
expect(await komodoEnsureTag('t5')).toMatchObject({ ok: false });
|
||||
});
|
||||
});
|
||||
@@ -1,29 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
||||
});
|
||||
afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); });
|
||||
|
||||
describe('Komodo startup response classification', () => {
|
||||
it.each([403, 502])('preserves HTTP %i so callers can distinguish rejection from uncertain startup', async httpStatus => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response('Deployment response failed', { status: httpStatus })));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus });
|
||||
});
|
||||
|
||||
it('identifies an explicit success=false response as a confirmed HTTP 200 failure', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: false, message: 'Image rejected' })));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus: 200, message: 'Image rejected' });
|
||||
});
|
||||
|
||||
it('returns a structured unreachable result when the startup response times out', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => { throw new DOMException('Request aborted', 'AbortError'); }));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'unreachable', message: expect.stringContaining('timed out') });
|
||||
});
|
||||
});
|
||||
@@ -1,148 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js';
|
||||
|
||||
describe('container readiness', () => {
|
||||
it.each([
|
||||
['Up 35 seconds (health: starting)', 'starting'],
|
||||
['Up about a minute (healthy)', 'running'],
|
||||
['Up 5 minutes (unhealthy)', 'unhealthy'],
|
||||
['Up 2 hours', 'running'],
|
||||
['Exited (1) 5 seconds ago', 'exited']
|
||||
])('reads nested Komodo container status: %s', (status, expected) => {
|
||||
expect(extractContainerState({ state: 'Running', container: { state: 'running', status } })).toBe(expected);
|
||||
});
|
||||
it('reads lowercase inspect health and does not trust missing containers or configuration', () => {
|
||||
expect(extractContainerState({ state: { status: 'running', running: true, health: { status: 'starting' } } })).toBe('starting');
|
||||
expect(extractContainerState({ state: 'Running', container: null })).toBe('unknown');
|
||||
expect(extractContainerState({ config: { state: 'running' }, status: 'ok' })).toBeNull();
|
||||
expect(extractContainerState({ state: 'Exited', container: { status: 'Up 1 minute (healthy)' } })).toBe('exited');
|
||||
});
|
||||
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
|
||||
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
|
||||
expect(extractContainerState({ state: 'running', health })).toBe(health);
|
||||
expect(extractContainerState({ state: 'running', health: { status: health } })).toBe(health);
|
||||
});
|
||||
it('accepts healthy running containers and preserves stopped states', () => {
|
||||
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running');
|
||||
expect(extractContainerState({ State: { Status: 'exited', Running: false, Health: { Status: 'healthy' } } })).toBe('exited');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractUdpPort', () => {
|
||||
it('returns the published port from docker-compose-style object entries', () => {
|
||||
const payload = {
|
||||
state: {
|
||||
container: {
|
||||
ports: [
|
||||
{ protocol: 'tcp', published: 3000, target: 3000 },
|
||||
{ protocol: 'udp', published: 4976, target: 28960 }
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
expect(extractUdpPort(payload)).toBe(4976);
|
||||
});
|
||||
|
||||
it('parses docker-run-style "host:container/proto" strings', () => {
|
||||
const payload = { ports: ['3074:3074/tcp', '4977:28960/udp'] };
|
||||
expect(extractUdpPort(payload)).toBe(4977);
|
||||
});
|
||||
|
||||
it('picks the UDP entry even when TCP appears first', () => {
|
||||
const payload = {
|
||||
deeply: {
|
||||
nested: {
|
||||
ports: [
|
||||
{ Protocol: 'tcp', PublishedPort: 80 },
|
||||
{ Protocol: 'udp', PublishedPort: 5555 }
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
expect(extractUdpPort(payload)).toBe(5555);
|
||||
});
|
||||
|
||||
it('returns null when no UDP mapping exists', () => {
|
||||
const payload = { ports: [{ protocol: 'tcp', published: 3000 }] };
|
||||
expect(extractUdpPort(payload)).toBeNull();
|
||||
});
|
||||
|
||||
it('survives cyclic references', () => {
|
||||
const cyc: Record<string, unknown> = {};
|
||||
cyc.self = cyc;
|
||||
expect(extractUdpPort(cyc)).toBeNull();
|
||||
});
|
||||
|
||||
it('recognizes HostPort / hostPort / host_port on object entries', () => {
|
||||
expect(extractUdpPort({ ports: [{ protocol: 'udp', HostPort: 28962 }] })).toBe(28962);
|
||||
expect(extractUdpPort({ ports: [{ protocol: 'udp', hostPort: 28963 }] })).toBe(28963);
|
||||
expect(extractUdpPort({ ports: [{ protocol: 'udp', host_port: 28964 }] })).toBe(28964);
|
||||
});
|
||||
|
||||
it('recognizes `type` alongside `protocol` as the protocol field', () => {
|
||||
expect(extractUdpPort({ ports: [{ type: 'UDP', published: 28965 }] })).toBe(28965);
|
||||
});
|
||||
|
||||
it('parses host-network "container/udp" strings', () => {
|
||||
expect(extractUdpPort({ ports: ['28966/udp'] })).toBe(28966);
|
||||
});
|
||||
|
||||
it('handles docker-inspect container-keyed PortBindings shape', () => {
|
||||
const payload = {
|
||||
NetworkSettings: {
|
||||
Ports: {
|
||||
'28960/udp': [{ HostIp: '0.0.0.0', HostPort: '28967' }],
|
||||
'3074/tcp': [{ HostIp: '0.0.0.0', HostPort: '3074' }]
|
||||
}
|
||||
}
|
||||
};
|
||||
expect(extractUdpPort(payload)).toBe(28967);
|
||||
});
|
||||
|
||||
it('prefers `ports`-keyed arrays over unrelated arrays discovered deeper', () => {
|
||||
// The unrelated array happens to contain a matching entry — the walker
|
||||
// must still prefer the real `ports` list.
|
||||
const payload = {
|
||||
labels: [{ protocol: 'udp', published: 9999 }],
|
||||
state: { container: { ports: [{ protocol: 'udp', published: 28968 }] } }
|
||||
};
|
||||
expect(extractUdpPort(payload)).toBe(28968);
|
||||
});
|
||||
|
||||
it('splits Komodo`s newline-delimited string form of `config.ports`', () => {
|
||||
// Verbatim shape from a real Komodo GetDeployment response — the
|
||||
// container publishes UDP 28963 and TCP 28963 as a single string.
|
||||
const payload = {
|
||||
name: 't5-nacht-1',
|
||||
config: {
|
||||
image: { type: 'Image', params: { image: 'ghcr.io/ayymoss/plutainer:latest' } },
|
||||
network: 'komodo_default',
|
||||
ports: '28963:28963/udp\n28963:28963/tcp\n'
|
||||
}
|
||||
};
|
||||
expect(extractUdpPort(payload)).toBe(28963);
|
||||
});
|
||||
|
||||
it('handles a single-line string-form ports value', () => {
|
||||
expect(extractUdpPort({ config: { ports: '28969:28960/udp' } })).toBe(28969);
|
||||
});
|
||||
|
||||
it('handles a host-network string-form ports value (no host prefix)', () => {
|
||||
expect(extractUdpPort({ config: { ports: '28970/udp\n' } })).toBe(28970);
|
||||
});
|
||||
|
||||
it('ignores TCP-only string-form ports entries', () => {
|
||||
expect(extractUdpPort({ config: { ports: '80:80/tcp\n443:443/tcp' } })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('komodoResolvePortDetailed', () => {
|
||||
it('is importable and has the discriminated-union shape at the type level', async () => {
|
||||
// Runtime import shim just to prove it's exported. Behavior is
|
||||
// covered by the fixture-based extractUdpPort tests above plus manual
|
||||
// end-to-end verification against a live Komodo.
|
||||
const mod = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(typeof mod.komodoResolvePortDetailed).toBe('function');
|
||||
expect(typeof mod.komodoResolveGamePort).toBe('function');
|
||||
});
|
||||
});
|
||||
@@ -1,17 +1,9 @@
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeploy: vi.fn(async () => ({ ok: true })),
|
||||
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
||||
@@ -27,11 +19,9 @@ const input = { ...selection, gameMode: 'zombie' as const, plutoKey: 'private-pl
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
await prepareNativeHost();
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'private-api-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', 'private-rcon-password');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
@@ -46,42 +36,11 @@ async function seedArtifacts() {
|
||||
}
|
||||
|
||||
describe('launch recipes and previews', () => {
|
||||
it('keeps detail polling independent of retained plugin bytes after launching', async () => {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const plugin = createScript({ name: 'Large utility plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
|
||||
targetRelPath: 'large.dll', content: Buffer.alloc(8 * 1024 ** 2, 42) });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(plugin.version.id, plugin.script.id);
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
const queuedSummary = getDeployment(queued.deploymentId)!.recipe!;
|
||||
await processCreations();
|
||||
const full = recipeForDeployment(queued.deploymentId)!.recipe;
|
||||
const expected = summarizeRecipe(full);
|
||||
expect(expected.fingerprint).not.toBe(queuedSummary.fingerprint);
|
||||
expect(full.files.find(file => file.path === 'large.dll')?.content).toBe(plugin.version.content.toString('base64'));
|
||||
closeDb();
|
||||
const parse = JSON.parse;
|
||||
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
|
||||
if (value.length > 1024 * 1024) throw new Error('Detail polling parsed plugin bytes.');
|
||||
return parse(value, reviver);
|
||||
});
|
||||
try {
|
||||
for (let read = 0; read < 3; read++) expect(getDeployment(queued.deploymentId)?.recipe).toEqual(expected);
|
||||
const row = db().prepare('SELECT length(recipe_summary) AS bytes FROM server_launch_recipes WHERE deployment_id = ?')
|
||||
.get(queued.deploymentId) as { bytes: number };
|
||||
expect(row.bytes).toBeLessThan(10_000);
|
||||
} finally { parsing.mockRestore(); }
|
||||
});
|
||||
|
||||
it('previews without reserving capacity, saving keys, exposing bytes, or calling Komodo', async () => {
|
||||
it('previews without reserving capacity, saving keys, exposing bytes, or submitting host commands', async () => {
|
||||
await seedArtifacts();
|
||||
const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = previewCreation(selection);
|
||||
const second = previewCreation(selection);
|
||||
expect(first).toMatchObject({ name: 't5_zombie_theater_1', port: 28960, hasSavedKey: false, ready: true });
|
||||
@@ -91,68 +50,9 @@ describe('launch recipes and previews', () => {
|
||||
expect(first.recipe.files[0]).not.toHaveProperty('content');
|
||||
expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(first)).not.toContain('private-');
|
||||
});
|
||||
|
||||
it('uses promotions made after review and queueing, including plugins and composed configuration', async () => {
|
||||
const { baseline, script } = await seedArtifacts();
|
||||
const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const plugin = createScript({ name: 'Game utilities', kind: 'plugin', scopeType: 'game', scopeValue: 't5', targetRelPath: 'utilities.dll', content: Buffer.from('old plugin') });
|
||||
const overlay = createScript({ name: 'Kino settings', kind: 'config', scopeType: 'map', scopeValue: 'zombie_theater', content: Buffer.from('set g_speed "180"\n') });
|
||||
for (const item of [plugin, overlay]) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(item.version.id, item.script.id);
|
||||
const preview = previewCreation(selection);
|
||||
const newScript = uploadNewVersion(script.script.id, Buffer.from('main(){ new_perks(); }'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newScript.id, script.script.id);
|
||||
const newOverlay = uploadNewVersion(overlay.script.id, Buffer.from('set g_speed "200"\n'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newOverlay.id, overlay.script.id);
|
||||
uploadNewVersion(script.script.id, Buffer.from('main(){ unpromoted_draft(); }'));
|
||||
const queued = queueCreateServer({ ...input, previewId: preview.id });
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
expect(recipeForDeployment(queued.deploymentId)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newScript.id }));
|
||||
const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newBaseline.id, baseline.script.id);
|
||||
const newPlugin = uploadNewVersion(plugin.script.id, Buffer.from('new plugin'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newPlugin.id, plugin.script.id);
|
||||
uploadNewVersion(plugin.script.id, Buffer.from('unpromoted plugin'));
|
||||
closeDb();
|
||||
await processCreations();
|
||||
expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('main(){ new_perks(); }');
|
||||
const deployedConfig = await readFile(join(fixture.t5, queued.name, 'configs/dedicated_sp.cfg'), 'utf8');
|
||||
expect(deployedConfig).toContain('set sv_voice "0"');
|
||||
expect(deployedConfig).toContain('set g_speed "200"');
|
||||
expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/plugins/utilities.dll'), 'utf8')).toBe('new plugin');
|
||||
expect(getDeployment(queued.deploymentId)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'utilities.dll', versionId: newPlugin.id }));
|
||||
expect(getServerScript(queued.name, 'perks.gsc')).toMatchObject({ script_version_id: newScript.id, sha256: newScript.sha256 });
|
||||
expect(getServerScript(queued.name, 'utilities.dll')).toMatchObject({ script_version_id: newPlugin.id, sha256: newPlugin.sha256 });
|
||||
const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg');
|
||||
if (recordedConfig?.content === undefined) throw new Error('Attempted launch must retain the configuration bytes.');
|
||||
expect(Buffer.from(recordedConfig.content, 'base64').toString()).toBe(deployedConfig);
|
||||
expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint);
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
await processCreations();
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
expect(JSON.stringify(getDeployment(queued.deploymentId))).not.toContain('private-');
|
||||
});
|
||||
|
||||
it('uses the latest promotion at provisioning for API clients without a preview', async () => {
|
||||
const { script } = await seedArtifacts();
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
const changed = uploadNewVersion(script.script.id, Buffer.from('changed'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(changed.id, script.script.id);
|
||||
await processCreations();
|
||||
expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('changed');
|
||||
});
|
||||
|
||||
it('returns the same operation for repeated submission, even after expiry and restart', async () => {
|
||||
const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -167,31 +67,6 @@ describe('launch recipes and previews', () => {
|
||||
expect(getServerKey(preview.name!)).toBe('private-pluto-key');
|
||||
});
|
||||
|
||||
it('rejects stale slots and ports before saving a key or queueing a second server', async () => {
|
||||
const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const preview = previewCreation(selection);
|
||||
expect(queueCreateServer({ ...input, mapSlug: 'moon' }).ok).toBe(true);
|
||||
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'no-slot' });
|
||||
expect(getServerKey(preview.name!)).toBeUndefined();
|
||||
const fresh = previewCreation(selection);
|
||||
expect(fresh.port).toBe(28961);
|
||||
expect(queueCreateServer({ ...input, previewId: fresh.id }).ok).toBe(true);
|
||||
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'no-slot' });
|
||||
expect(getServerKey('t5_zombie_theater_2')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not provision an already accepted preview again through internal callers', async () => {
|
||||
const { previewCreation, createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const preview = previewCreation(selection);
|
||||
const first = await createServer({ ...input, previewId: preview.id });
|
||||
expect(first.ok).toBe(true);
|
||||
expect(await createServer({ ...input, previewId: preview.id })).toEqual(first);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('carries preview identity through the public creation API and rejects malformed requests', async () => {
|
||||
const { POST: preview } = await import('../../src/routes/api/deployments/preview/+server.js');
|
||||
const { POST: launch } = await import('../../src/routes/api/servers/+server.js');
|
||||
@@ -218,93 +93,6 @@ describe('launch recipes and previews', () => {
|
||||
expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports configuration blockers while still allowing recipes to be saved', async () => {
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', ''); vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const preview = previewCreation(selection);
|
||||
expect(preview.ready).toBe(false);
|
||||
expect(preview.checks.filter(c => c.state === 'blocked')).toHaveLength(2);
|
||||
expect(saveBlueprint({ name: 'For later', previewId: preview.id }).name).toBe('For later');
|
||||
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'unconfigured' });
|
||||
});
|
||||
|
||||
it('resolves current file eligibility after a blueprint review and queued launch', async () => {
|
||||
const { script } = await seedArtifacts();
|
||||
const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveBlueprint, deleteBlueprint, listBlueprints, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const removed = createScript({ name: 'Removed script', targetRelPath: 'removed.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('removed') });
|
||||
const unpromoted = createScript({ name: 'Unpromoted script', targetRelPath: 'unpromoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('unpromoted') });
|
||||
for (const item of [removed, unpromoted]) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(item.version.id, item.script.id);
|
||||
const blueprint = saveBlueprint({ name: 'Classic night', description: 'Kino lineup', previewId: previewCreation(selection).id });
|
||||
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string };
|
||||
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries: [{ count: 1, ...selection }] });
|
||||
closeDb();
|
||||
expect(listBlueprints()[0]).toEqual(blueprint);
|
||||
const preview = previewCreation({ blueprintId: blueprint.id });
|
||||
deleteBlueprint(blueprint.id);
|
||||
const queued = queueCreateServer({ ...input, previewId: preview.id });
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
db().prepare('UPDATE scripts SET current_version_id = NULL WHERE id = ?').run(removed.script.id);
|
||||
db().prepare('DELETE FROM scripts WHERE id = ?').run(removed.script.id);
|
||||
db().prepare('UPDATE scripts SET current_version_id = NULL WHERE id = ?').run(unpromoted.script.id);
|
||||
setScope({ scriptId: script.script.id, scopeType: 'map', scopeValue: 'zombie_moon' });
|
||||
const added = createScript({ name: 'New telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'map', scopeValue: 'zombie_theater', content: Buffer.from('newly promoted telemetry') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(added.version.id, added.script.id);
|
||||
createScript({ name: 'Draft telemetry', targetRelPath: 'draft.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('never promoted') });
|
||||
await processCreations();
|
||||
for (const file of ['perks.gsc', 'removed.gsc', 'unpromoted.gsc', 'draft.gsc']) {
|
||||
await expect(readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp', file))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
}
|
||||
expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('newly promoted telemetry');
|
||||
expect(recipeForDeployment(queued.deploymentId)?.blueprintName).toBe('Classic night');
|
||||
expect(recipeForDeployment(queued.deploymentId)?.recipe.files.filter(file => file.kind === 'script')).toEqual([
|
||||
expect.objectContaining({ path: 'telemetry.gsc', versionId: added.version.id })
|
||||
]);
|
||||
});
|
||||
|
||||
it('launches again with current promotions while preserving the completed deployment record', async () => {
|
||||
const { script } = await seedArtifacts();
|
||||
const { previewCreation, createServer, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const original = await createServer(input);
|
||||
if (!original.ok) throw new Error(original.message);
|
||||
const historicalRecipe = recipeForDeployment(original.deploymentId);
|
||||
const promoted = uploadNewVersion(script.script.id, Buffer.from('current telemetry'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(promoted.id, script.script.id);
|
||||
const replay = previewCreation({ deploymentId: original.deploymentId });
|
||||
expect(replay).toMatchObject({ name: 't5_zombie_theater_2', port: 28961, hasSavedKey: false });
|
||||
expect(replay.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: promoted.id }));
|
||||
const blueprint = saveBlueprint({ name: 'Restored', deploymentId: original.deploymentId });
|
||||
expect(blueprint.entries).toEqual([{ count: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino Der Toten' }]);
|
||||
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string };
|
||||
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries: [{ count: 1, ...selection }] });
|
||||
expect(previewCreation({ blueprintId: blueprint.id }).recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: promoted.id }));
|
||||
const queued = queueCreateServer({ ...input, previewId: replay.id });
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
await processCreations();
|
||||
expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('current telemetry');
|
||||
expect(recipeForDeployment(original.deploymentId)).toEqual(historicalRecipe);
|
||||
expect(await readFile(join(fixture.t5, original.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('main(){ old_perks(); }');
|
||||
});
|
||||
|
||||
it('preserves the exact T6 map variant in a blueprint', async () => {
|
||||
const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const preview = previewCreation({ game: 't6', mapSlug: 'town' });
|
||||
const blueprint = saveBlueprint({ name: 'Town survival', previewId: preview.id });
|
||||
const saved = previewCreation({ blueprintId: blueprint.id });
|
||||
expect(saved.recipe).toMatchObject({ game: 't6', mapSlug: 'town', expectedMap: 'zm_transit', configFile: 'dedicated_zm.cfg' });
|
||||
const queued = queueCreateServer({ ...input, game: 't6', mapSlug: 'town', previewId: saved.id });
|
||||
if (!queued.ok) throw new Error(queued.message);
|
||||
await processCreations();
|
||||
expect(await readFile(join(fixture.t6, queued.name, 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
|
||||
});
|
||||
|
||||
it('validates blueprint names, sources, and case-insensitive duplicates', async () => {
|
||||
const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
@@ -316,38 +104,3 @@ describe('launch recipes and previews', () => {
|
||||
expect(() => saveBlueprint({ name: ' classic night ', previewId })).toThrow(/already exists/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resuming verification', () => {
|
||||
it('resumes an interrupted creation without creating or restarting its container again', async () => {
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { interruptDeployments, getDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const created = await createServer(input);
|
||||
if (!created.ok) throw new Error(created.message);
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(created.deploymentId);
|
||||
interruptDeployments();
|
||||
expect(getDeployment(created.deploymentId)?.canResumeVerification).toBe(true);
|
||||
expect(resumeVerification(created.deploymentId)).toBe(true);
|
||||
expect(resumeVerification(created.deploymentId)).toBe(false);
|
||||
await processCreations();
|
||||
expect(getDeployment(created.deploymentId)?.deployment).toMatchObject({ state: 'done', error: null });
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('refuses to observe a replacement server or interfere with a later destruction', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { finishDeployment, resumeVerification, getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const created = await createServer(input);
|
||||
if (!created.ok) throw new Error(created.message);
|
||||
finishDeployment(created.deploymentId, { ok: false, message: 'Interrupted' });
|
||||
db().prepare('UPDATE servers SET game_port = game_port + 1 WHERE name = ?').run(created.name);
|
||||
expect(resumeVerification(created.deploymentId)).toBe(false);
|
||||
db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(created.port, created.name);
|
||||
expect(getDeployment(created.deploymentId)?.canResumeVerification).toBe(true);
|
||||
destroyServer(created.name);
|
||||
expect(resumeVerification(created.deploymentId)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -122,8 +122,8 @@ it('migrates legacy participation as unknown before snapshot backfill and preser
|
||||
seed(handle, 1, { status: 'active' });
|
||||
handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN first_observed_round');
|
||||
handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN present_at_start');
|
||||
handle.prepare(`INSERT INTO servers (name, komodo_name, game, map)
|
||||
VALUES ('t5_zombie_moon_1', 'moon', 't5', 'zombie_moon')`).run();
|
||||
handle.prepare(`INSERT INTO servers (name, game, map)
|
||||
VALUES ('t5_zombie_moon_1', 't5', 'zombie_moon')`).run();
|
||||
const saved: GameTelemetry = { event: 'game_end', server: 't5_zombie_moon_1', map: 'zombie_moon', round: 8,
|
||||
players: [player('202', 80)], receivedAt: timestamp(2) };
|
||||
handle.prepare('INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)')
|
||||
|
||||
@@ -11,7 +11,7 @@ const cfgPath = (server: string) => join(fixture.t5, server, 'configs', 'dedicat
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', '');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
|
||||
@@ -15,8 +15,8 @@ beforeEach(async () => {
|
||||
received = 0;
|
||||
fixture = await newFixture([]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO servers (name, komodo_name, game, map) VALUES (?, ?, 't5', 'zombie_theater')")
|
||||
.run(server, server);
|
||||
db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't5', 'zombie_theater')")
|
||||
.run(server);
|
||||
});
|
||||
afterEach(async () => fixture.cleanup());
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('completed match details', () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
const deleted = getMatchDetail(1)!;
|
||||
expect(deleted).toEqual({ ...before, server_deleted_at: expect.any(String) });
|
||||
db().prepare("INSERT INTO servers (name, komodo_name, game, map) VALUES (?, 'replacement', 't6', 'zm_transit')").run(server);
|
||||
db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't6', 'zm_transit')").run(server);
|
||||
const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js');
|
||||
recordLeaderboardTelemetry(db(), { event: 'game_end', server, map: 'zm_transit', round: 20,
|
||||
players: [player('999', { kills: 999 })], receivedAt: timestamp(4) });
|
||||
|
||||
@@ -64,8 +64,8 @@ describe('historical metrics', () => {
|
||||
const at = Date.UTC(2026, 0, 2);
|
||||
expect(readMetrics(1, at)).toMatchObject({ firstRetainedSampleAt: null, latestSampleAt: null,
|
||||
registeredServerCount: 0, servers: [], maps: [] });
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('current', 'current', null);
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('removed', 'removed', new Date(at).toISOString());
|
||||
db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('current', null);
|
||||
db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('removed', new Date(at).toISOString());
|
||||
const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null };
|
||||
recordMetrics([sample], at);
|
||||
const emptyRange = readMetrics(1, at + 2 * 86400_000);
|
||||
@@ -341,8 +341,8 @@ describe('historical metrics', () => {
|
||||
recordMetrics([sample, { ...sample, server: 'removed' }, { ...sample, server: 'tombstone' }], at);
|
||||
// A newly registered name can link to the current server without asserting
|
||||
// that its earlier historical observations belong to this registration.
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run('reused', 'reused', new Date(at + 15_000).toISOString());
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('tombstone', 'tombstone', new Date(at).toISOString());
|
||||
db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run('reused', new Date(at + 15_000).toISOString());
|
||||
db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('tombstone', new Date(at).toISOString());
|
||||
const result = readMetrics(1, at + 30_000);
|
||||
expect(result.servers.find(server => server.server === 'reused')!.currentRegistered).toBe(true);
|
||||
expect(result.servers.filter(server => server.server !== 'reused').every(server => !server.currentRegistered)).toBe(true);
|
||||
|
||||
@@ -10,7 +10,7 @@ const server = 't5_zombie_theater_1';
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
});
|
||||
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
@@ -30,12 +30,12 @@ async function intent() {
|
||||
}
|
||||
|
||||
describe('native creation intents before placement', () => {
|
||||
it('retains its identity and native history across restart without falling into legacy provisioning', async () => {
|
||||
it('retains its identity and native history across restart without changing ownership', async () => {
|
||||
const id = await intent();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
|
||||
const { nextLobbyNumber, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
expect(nativeDeployment(id)).toBe(true);
|
||||
expect(unplacedNativeCreation(server)).toBe(id);
|
||||
@@ -43,7 +43,8 @@ describe('native creation intents before placement', () => {
|
||||
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
await processCreations();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(getDeployment(id)).toMatchObject({ backend: 'native', canCancelCreation: true, agentOperation: null,
|
||||
deployment: { state: 'queued', server_exists: 0 } });
|
||||
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null });
|
||||
@@ -78,7 +79,7 @@ describe('native creation intents before placement', () => {
|
||||
it('will not cancel an unplaced historical intent against a replacement server', async () => {
|
||||
const id = await intent();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO servers(name,komodo_name) VALUES(?,'replacement')").run(server);
|
||||
db().prepare("INSERT INTO servers(name) VALUES(?)").run(server);
|
||||
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
expect(canCancelCreation(id)).toBe(false);
|
||||
expect(cancelCreation(id)).toBeNull();
|
||||
|
||||
@@ -22,8 +22,8 @@ async function setup(capable = true) {
|
||||
});
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const database = db();
|
||||
database.prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,created_by)
|
||||
VALUES(?,'','t5','zombie_theater','zombie','theater','ui')`).run(server);
|
||||
database.prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,created_by)
|
||||
VALUES(?,'t5','zombie_theater','zombie','theater','ui')`).run(server);
|
||||
database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES(?,?,?,'library','image',28960,?)`).run(randomUUID(), server, host.id, new Date().toISOString());
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ beforeEach(async () => {
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id;
|
||||
const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3,
|
||||
|
||||
@@ -10,7 +10,7 @@ async function setup() {
|
||||
const host = hosts.createHost({ name: 'Observation host' });
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO servers(name,komodo_name) VALUES('test','')").run();
|
||||
db().prepare("INSERT INTO servers(name) VALUES('test')").run();
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('instance','test',?,'library','image',28960,?)`).run(host.id, new Date().toISOString());
|
||||
const native = await import('../../src/lib/server/backends/native.js');
|
||||
|
||||
@@ -19,7 +19,7 @@ beforeEach(async () => {
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id;
|
||||
enrollHost(createEnrollment(hostId).token, inventory());
|
||||
@@ -250,7 +250,6 @@ describe('native instance provisioning', () => {
|
||||
expect(result.progress?.checks[0].source).toBe('agent');
|
||||
const steps = provisioningGroups(result).flatMap(group => group.steps);
|
||||
expect(steps.every(step => step.state === 'done')).toBe(true);
|
||||
expect(steps.some(step => step.label.includes('Komodo'))).toBe(false);
|
||||
expect(result.agentOperation?.state).toBe('completed');
|
||||
});
|
||||
|
||||
@@ -410,7 +409,7 @@ describe('native instance provisioning', () => {
|
||||
expect(nativeInstance(job.name)?.library_id).toBe(acceptedId);
|
||||
});
|
||||
|
||||
it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => {
|
||||
it.each(['t5', 't6'] as const)('queues and verifies a %s deployment through its host agent', async game => {
|
||||
const job = await queue(game); await advance();
|
||||
const operation = await operationFor(job.deploymentId);
|
||||
expect(job.port).toBeNull();
|
||||
@@ -498,8 +497,8 @@ describe('native instance provisioning', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
||||
// Version evidence must follow the submitted command, even on a fast test host.
|
||||
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
|
||||
@@ -526,8 +525,8 @@ describe('native instance provisioning', () => {
|
||||
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
||||
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
||||
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
|
||||
db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
|
||||
it('retires only obsolete backend columns while preserving instances, history, keys and foreign keys', async () => {
|
||||
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
||||
const previous = new Database(fixture.dbPath);
|
||||
// Model the former required resource identity and an unrelated extension.
|
||||
previous.exec(schema);
|
||||
previous.exec(`ALTER TABLE servers ADD COLUMN komodo_type TEXT NOT NULL DEFAULT 'deployment';
|
||||
ALTER TABLE servers ADD COLUMN komodo_name TEXT NOT NULL;
|
||||
ALTER TABLE servers ADD COLUMN komodo_server TEXT;
|
||||
ALTER TABLE servers ADD COLUMN operator_note TEXT;`);
|
||||
previous.prepare(`INSERT INTO servers(name,komodo_name,komodo_server,game,game_port,operator_note)
|
||||
VALUES('native-server','','old-host','t5',28960,'keep this')`).run();
|
||||
previous.prepare("INSERT INTO server_keys(server_id,pluto_key) VALUES('native-server','saved-key')").run();
|
||||
previous.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
||||
VALUES('creation','native-server','create','done','Ready','2026-01-01','fixture')`).run();
|
||||
previous.prepare(`INSERT INTO server_deployment_events(deployment_id,created_at,message)
|
||||
VALUES('creation','2026-01-01','Original receipt')`).run();
|
||||
const { migrateNativeInstances } = await import('../../src/lib/server/backends/schema.js');
|
||||
migrateNativeInstances(previous);
|
||||
previous.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES('instance','native-server','host','library','image',28960,'2026-01-01')`).run();
|
||||
previous.close();
|
||||
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
const handle = db();
|
||||
const columns = (handle.prepare('PRAGMA table_info(servers)').all() as { name: string }[]).map(column => column.name);
|
||||
expect(columns).not.toEqual(expect.arrayContaining(['komodo_type']));
|
||||
expect(columns).not.toContain('komodo_name');
|
||||
expect(columns).not.toContain('komodo_server');
|
||||
expect(columns).toContain('operator_note');
|
||||
expect(handle.prepare('SELECT name,game_port,operator_note FROM servers WHERE name=?').get('native-server'))
|
||||
.toEqual({ name: 'native-server', game_port: 28960, operator_note: 'keep this' });
|
||||
expect(handle.prepare('SELECT id,generation FROM native_instances').all()).toEqual([{ id: 'instance', generation: 1 }]);
|
||||
expect(handle.prepare('SELECT pluto_key FROM server_keys').get()).toEqual({ pluto_key: 'saved-key' });
|
||||
expect(handle.prepare('SELECT message FROM server_deployment_events').get()).toEqual({ message: 'Original receipt' });
|
||||
expect(handle.pragma('foreign_key_check')).toEqual([]);
|
||||
expect(handle.pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
closeDb();
|
||||
}
|
||||
// Native inserts must work without supplying the retired required field.
|
||||
expect(() => db().prepare("INSERT INTO servers(name) VALUES('new-native-server')").run()).not.toThrow();
|
||||
});
|
||||
@@ -13,14 +13,14 @@ const server = 't5_zombie_theater_1';
|
||||
const healthy = { exists: true, healthy: true, state: 'running' };
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]);
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
|
||||
const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
hostId = createHost({ name: 'Safety test host' }).id;
|
||||
enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }] });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
instanceId = randomUUID();
|
||||
db().prepare("INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'','t5','zombie_theater','zombie','theater',28960,'running')").run(server);
|
||||
db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server);
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`).run(instanceId, server, hostId, new Date().toISOString());
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
||||
@@ -31,8 +31,8 @@ beforeEach(async () => {
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
||||
|
||||
async function advance() {
|
||||
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
||||
await processNativeProvisioning(() => null);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
}
|
||||
async function operationId(id: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -119,3 +119,56 @@ describe('native disruption safety at dispatch', () => {
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('native scheduler ownership', () => {
|
||||
it('rejects lifecycle commands for a registry row without a host instance', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
db().prepare("INSERT INTO servers(name) VALUES('unplaced-registry')").run();
|
||||
expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({ ok: false, status: 503 });
|
||||
expect(destroyServer('unplaced-registry')).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('shares restart capacity until an agent receipt and fresh health release the first slot', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
|
||||
db().prepare("INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)").run();
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?`).run(instanceId);
|
||||
const first = await waitingRestart();
|
||||
const second = requestLifecycle('second', 'restart');
|
||||
if (!second.ok) throw new Error(second.error);
|
||||
await advance();
|
||||
const command = await operationId(first);
|
||||
expect(command).toEqual(expect.any(String));
|
||||
expect(await operationId(second.deploymentId)).toBeNull();
|
||||
await advance();
|
||||
expect(await operationId(first)).toBe(command);
|
||||
expect(await operationId(second.deploymentId)).toBeNull();
|
||||
reportHostOperation(hostId, command!, { state: 'completed', result: healthy });
|
||||
await advance();
|
||||
await advance();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({ state: 'done' });
|
||||
expect(await operationId(second.deploymentId)).toEqual(expect.any(String));
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => {
|
||||
const id = await stageCampaign();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'").run();
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'waiting' });
|
||||
db().prepare("UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'").run();
|
||||
await advance();
|
||||
expect(await operationId(id)).toBeNull();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,7 +10,7 @@ describe('persistent activity', () => {
|
||||
it('keeps beta failures and retries across reopening without duplicating polls', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
||||
handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run();
|
||||
handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run();
|
||||
handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run();
|
||||
@@ -29,7 +29,7 @@ describe('persistent activity', () => {
|
||||
it('records campaign files, restart and health milestones, and cascades cleanup', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
||||
handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')").run();
|
||||
handle.prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')").run();
|
||||
handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')").run();
|
||||
|
||||
@@ -5,7 +5,7 @@ let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture();
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
|
||||
+22
-236
@@ -3,7 +3,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
|
||||
let fixture: Fixture;
|
||||
@@ -13,15 +12,15 @@ beforeEach(async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
|
||||
fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
||||
vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
const k = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true });
|
||||
vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
||||
const k = await import('../../src/lib/server/backends/native.js');
|
||||
|
||||
vi.mocked(k.nativeHealth).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const r = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
@@ -38,18 +37,18 @@ describe('Plutonium versions', () => {
|
||||
const m = await api();
|
||||
await m.checkPlutonium();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString());
|
||||
closeDb();
|
||||
vi.mocked(komodoInspectHealthy).mockClear();
|
||||
vi.mocked(nativeHealth).mockClear();
|
||||
vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
vi.setSystemTime(Date.now() + 1_000);
|
||||
await m.checkPlutonium();
|
||||
expect(komodoInspectHealthy).toHaveBeenCalledTimes(1);
|
||||
expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]);
|
||||
expect(nativeHealth).toHaveBeenCalledTimes(1);
|
||||
expect(nativeHealth).toHaveBeenCalledWith(servers[1]);
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]);
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
|
||||
@@ -63,17 +62,17 @@ describe('Plutonium versions', () => {
|
||||
const m = await api();
|
||||
await m.checkPlutonium();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString());
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
|
||||
.run(servers[0], new Date().toISOString(), 'failed-restart');
|
||||
vi.mocked(komodoInspectHealthy).mockClear();
|
||||
vi.mocked(nativeHealth).mockClear();
|
||||
vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
await m.checkPlutonium();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]);
|
||||
expect(nativeHealth).not.toHaveBeenCalledWith(servers[0]);
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]);
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
|
||||
db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
|
||||
@@ -86,13 +85,13 @@ describe('Plutonium versions', () => {
|
||||
await m.checkPlutonium();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
|
||||
const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString());
|
||||
vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
if (phase === 'health') vi.mocked(nativeHealth).mockImplementationOnce(async () => {
|
||||
queue(); return { ok: true, state: 'running' };
|
||||
});
|
||||
else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
||||
@@ -105,7 +104,7 @@ describe('Plutonium versions', () => {
|
||||
});
|
||||
it('coalesces published-release checks without probing the fleet', async () => {
|
||||
const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const first = checkRelease();
|
||||
const second = checkRelease();
|
||||
@@ -113,7 +112,7 @@ describe('Plutonium versions', () => {
|
||||
await Promise.all([first, second]);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(release()).toMatchObject({ revision: 5354, error: null });
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(nativeHealth).not.toHaveBeenCalled();
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
await checkRelease();
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
@@ -266,7 +265,7 @@ describe('Plutonium versions', () => {
|
||||
// Both registrations have no lifecycle submission, so only their
|
||||
// registry identity distinguishes the old and replacement instances.
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now);
|
||||
db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run(servers[0], now);
|
||||
}
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
expect(await probeVersion(servers[0])).toBe(5354);
|
||||
@@ -314,103 +313,9 @@ describe('maintenance timezones', () => {
|
||||
expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00');
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z');
|
||||
});
|
||||
it('persists the zone and advances in local calendar days across DST', async () => {
|
||||
vi.setSystemTime(new Date('2026-03-07T10:59:00Z'));
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z');
|
||||
vi.setSystemTime(new Date('2026-03-07T11:00:00Z'));
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistent maintenance', () => {
|
||||
it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await m.maintenanceTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
const job = m.plutoniumStatus().jobs[0];
|
||||
expect(job.provisioning_id).toBeTruthy();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
// The source has not mirrored submission yet; cancellation must still
|
||||
// consult the authoritative operation, not its stale queued state.
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('queued');
|
||||
expect(m.cancelMaintenance(job.id)).toBe(false);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(Date.now() + 15_000); await processProvisioning();
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
});
|
||||
it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
expect(m.queueMaintenance([servers[0]])).toBe(1);
|
||||
expect(m.queueMaintenance([servers[0]])).toBe(0);
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
|
||||
await maintenanceTick(); await maintenanceTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
await Promise.all([maintenanceTick(), maintenanceTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
|
||||
});
|
||||
it('shares rolling capacity across servers and survives closing the database', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
|
||||
await maintenanceTick();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
|
||||
});
|
||||
it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
await maintenanceTick();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('does not retry an ambiguous restart timeout', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
|
||||
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('honors cancellation arriving during the player check', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
|
||||
});
|
||||
await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
|
||||
});
|
||||
it('holds maintenance while lifecycle work is queued', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
|
||||
await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
|
||||
});
|
||||
it('waits to attach legacy maintenance while another operation owns the server', async () => {
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -425,29 +330,6 @@ describe('persistent maintenance', () => {
|
||||
expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy();
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 });
|
||||
});
|
||||
it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => {
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const submittedAt = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state)
|
||||
VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt);
|
||||
if (linked) {
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
|
||||
}
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
|
||||
.run(servers[0], submittedAt, linked ? 'legacy-restart' : null);
|
||||
await m.maintenanceTick();
|
||||
const job = m.plutoniumStatus().jobs[0];
|
||||
expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' });
|
||||
expect(job.provisioning_id).toBeTruthy();
|
||||
if (linked) expect(job.provisioning_id).toBe('legacy-restart');
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id });
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -455,12 +337,12 @@ describe('persistent maintenance', () => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id);
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString());
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
expect(nativeHealth).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run();
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
@@ -493,18 +375,6 @@ describe('persistent maintenance', () => {
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' });
|
||||
});
|
||||
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
it('validates API requests without changing the schedule or queue', async () => {
|
||||
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
|
||||
const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', {
|
||||
@@ -521,19 +391,6 @@ describe('persistent maintenance', () => {
|
||||
expect(m.plutoniumStatus().schedule.enabled).toBe(true);
|
||||
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' });
|
||||
});
|
||||
it('runs a daily UTC schedule once and coalesces missed days', async () => {
|
||||
expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z');
|
||||
expect(() => nextMaintenanceAt('24:00')).toThrow();
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
expect(m.plutoniumStatus().schedule.enabled).toBe(false);
|
||||
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
|
||||
vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
|
||||
await maintenanceTick(); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
|
||||
m.saveMaintenanceSchedule(false, '06:00', 'UTC');
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Maintenance policies and runtime updates', () => {
|
||||
@@ -614,64 +471,6 @@ describe('Maintenance policies and runtime updates', () => {
|
||||
expect(m.queueUpdates([servers[0]])).toBe(1);
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' });
|
||||
});
|
||||
it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
|
||||
const m = await outdated();
|
||||
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false);
|
||||
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
|
||||
p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true);
|
||||
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
|
||||
p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning(); await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => {
|
||||
const m = await outdated();
|
||||
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]);
|
||||
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
const failed = m.plutoniumStatus().jobs[0];
|
||||
expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' });
|
||||
closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(1);
|
||||
expect(m.retryMaintenance(failed.id)).toBe(1);
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' });
|
||||
expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined();
|
||||
});
|
||||
it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => {
|
||||
const m = await outdated(); m.queueUpdates([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE servers SET desired_state = 'stopped'").run();
|
||||
await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
|
||||
expect(m.queueUpdates()).toBe(0);
|
||||
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true);
|
||||
});
|
||||
it('separates scheduled restart success from revision verification and coalesces paused runs', async () => {
|
||||
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
|
||||
const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true);
|
||||
vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(0);
|
||||
p.pauseMaintenance(false); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
||||
expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true);
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' });
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done');
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
|
||||
});
|
||||
it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
|
||||
const m = await outdated();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -679,25 +478,12 @@ describe('Maintenance policies and runtime updates', () => {
|
||||
expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
|
||||
vi.setSystemTime(Date.now() + 3_600_000);
|
||||
expect(() => m.queueUpdates()).toThrow('published revision');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
vi.mocked(nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
await m.checkPlutonium();
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true);
|
||||
});
|
||||
it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => {
|
||||
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
|
||||
m.queueMaintenance([servers[0]]); await maintenanceTick();
|
||||
const before = m.plutoniumStatus();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb();
|
||||
const after = m.plutoniumStatus();
|
||||
expect(after.schedule).toEqual(before.schedule);
|
||||
expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' });
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,7 +32,7 @@ describe('provisioning cancellation', () => {
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
db().prepare("INSERT INTO servers(name,komodo_name) VALUES('one','one'),('two','two')").run();
|
||||
db().prepare("INSERT INTO servers(name) VALUES('one'),('two')").run();
|
||||
const id = beginDeployment('one','restart');
|
||||
const other = beginDeployment('two','restart');
|
||||
db().prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('one','2026-10-05',?),('two','2026-10-05',?)").run(id, other);
|
||||
|
||||
@@ -22,9 +22,9 @@ describe('provisioning detail projection', () => {
|
||||
|
||||
it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => {
|
||||
const history = await import('../../src/lib/server/provisioning/history.js');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const backend = await import('../../src/lib/server/backends/native.js');
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const inspect = vi.spyOn(komodo, 'komodoInspectHealthy');
|
||||
const inspect = vi.spyOn(backend, 'nativeHealth');
|
||||
const status = vi.spyOn(rcon, 'rconStatusResult');
|
||||
const id = history.beginDeployment('t5_zombie_theater_1', 'create');
|
||||
for (let index = 0; index < 40; index++) history.recordDeploymentEvent(id, `Recorded activity ${index}`);
|
||||
@@ -73,7 +73,7 @@ describe('provisioning detail projection', () => {
|
||||
it('correlates reservations by operation ID without assigning a later same-name slot to historical work', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment, getDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
db().prepare("INSERT INTO servers (name,komodo_name) VALUES ('server','server')").run();
|
||||
db().prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
||||
const old = beginDeployment('server', 'restart');
|
||||
finishDeployment(old, { ok: false, message: 'Timeout' });
|
||||
const current = beginDeployment('server', 'restart');
|
||||
@@ -104,7 +104,7 @@ describe('provisioning detail projection', () => {
|
||||
it('preserves a failure outcome and evidence before an observation-only same-ID resume', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const history = await import('../../src/lib/server/provisioning/history.js');
|
||||
db().prepare("INSERT INTO servers (name,komodo_name,game_port) VALUES ('server','server',28960)").run();
|
||||
db().prepare("INSERT INTO servers (name,game_port) VALUES ('server',28960)").run();
|
||||
const id = history.beginDeployment('server', 'create');
|
||||
db().prepare("INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')").run(id);
|
||||
history.deploymentStage(id, 'Waiting for game evidence', { phase: 'verify-game', checks: [
|
||||
|
||||
@@ -1,405 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_1';
|
||||
const status = (map = 'zombie_theater') => ({ ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map, gametype: 'zom', roster: [] } });
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354 })));
|
||||
fixture = await newFixture([server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
||||
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
}
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(status());
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function register() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
}
|
||||
async function detail(id: string) {
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
return getDeployment(id)!;
|
||||
}
|
||||
async function telemetry(players: number, map = 'zombie_theater') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const receivedAt = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
||||
ON CONFLICT(server) DO UPDATE SET players=excluded.players,received_at=excluded.received_at,payload=excluded.payload`)
|
||||
.run(server, players, receivedAt, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt }));
|
||||
return receivedAt;
|
||||
}
|
||||
|
||||
async function launch(game: 't5' | 't6' = 't5') {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockResolvedValue(status(game === 't5' ? 'zombie_theater' : 'zm_transit'));
|
||||
const job = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
await processCreations();
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
return { ...job, processCreations };
|
||||
}
|
||||
|
||||
describe('Plutonium provisioning verification', () => {
|
||||
it.each(['t5', 't6'] as const)('verifies production and persists the running %s revision before completing', async game => {
|
||||
const job = await launch(game);
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { PLUTONIUM_MANIFEST_URL } = await import('../../src/lib/plutonium.js');
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
await job.processCreations();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment.state).toBe('done');
|
||||
expect(completed.progress).toMatchObject({ phase: 'verify-plutonium', blocker: null });
|
||||
expect(completed.progress?.checks).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ key: 'plutonium-release', source: 'manifest', result: 'pass', actual: 5354 }),
|
||||
expect.objectContaining({ key: 'plutonium-version', source: 'rcon', result: 'pass', expected: 5354, actual: 5354 })
|
||||
]));
|
||||
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5354, update_status: 'current' });
|
||||
expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.objectContaining({ redirect: 'error' }));
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledWith(job.name);
|
||||
const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
|
||||
expect(isProvisioningDetail(completed, job.deploymentId)).toBe(true);
|
||||
expect(provisioningGroups(completed).find(group => group.key === 'verify')?.state).toBe('done');
|
||||
});
|
||||
|
||||
it('retries unknown runtime evidence while retaining startup capacity and never replays creation', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const next = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second-key' });
|
||||
if (!next.ok) throw new Error(next.message);
|
||||
// A pre-start observation is not proof of the newly started process.
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO plutonium_versions (server, revision, checked_at) VALUES (?, 5354, ?)')
|
||||
.run(job.name, '2026-09-28T11:59:00.000Z');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version reply timed out.' });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect((await detail(job.deploymentId)).progress?.blocker).toMatchObject({ code: 'plutonium-version', automatic: true });
|
||||
expect((await detail(next.deploymentId)).deployment.state).toBe('queued');
|
||||
expect(db().prepare('SELECT revision FROM plutonium_versions WHERE server = ?').get(job.name)).toEqual({ revision: null });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
expect((await detail(next.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('requires a successful manifest check even with a previously cached revision, then resumes without recreating', async () => {
|
||||
const job = await launch();
|
||||
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
|
||||
writeSetting('plutonium_release', { revision: 5300, checkedAt: new Date().toISOString(), error: null });
|
||||
vi.mocked(fetch).mockRejectedValueOnce(new Error('Manifest offline'));
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect((await detail(job.deploymentId)).progress?.checks).toContainEqual(expect.objectContaining({
|
||||
key: 'plutonium-release', result: 'unknown', actual: null, message: 'Manifest offline'
|
||||
}));
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reports a confirmed outdated runtime without automatically restarting or recreating it', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
||||
await job.processCreations();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment).toMatchObject({ state: 'failed', error: expect.stringContaining('running r5300, production is r5354') });
|
||||
expect(failed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'plutonium-version', result: 'fail', expected: 5354, actual: 5300 }));
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5300, update_status: 'outdated' });
|
||||
await job.processCreations();
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('accepts a runtime ahead of the published release without requesting a downgrade', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5355, error: null });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === job.name)?.update_status).toBe('ahead');
|
||||
});
|
||||
|
||||
it('discards a late version response when the instance was removed during the probe', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(job.name);
|
||||
return { revision: 5354, error: null };
|
||||
});
|
||||
await job.processCreations();
|
||||
const removed = await detail(job.deploymentId);
|
||||
expect(removed.deployment.server_deleted_at).toBeTruthy();
|
||||
expect(removed.deployment.state).not.toBe('done');
|
||||
expect(removed.progress?.checkpoints.some(check => check.key === 'verify-plutonium')).toBe(false);
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(job.name)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('provisioning decision evidence', () => {
|
||||
it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
|
||||
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { script } = createScript({ name: 'New promotion', targetRelPath: 'promoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('promoted while queued') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress?.recipeStatus).toBe('attempted');
|
||||
expect((await detail(job.deploymentId)).recipe?.files.some(file => file.path === 'promoted.gsc')).toBe(true);
|
||||
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce(status('zombie_moon'));
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'verify-map', blocker: { code: 'map-mismatch' },
|
||||
checks: [{ key: 'container', result: 'pass' }, { key: 'game', result: 'pass' }, { key: 'map', result: 'waiting', expected: 'zombie_theater', actual: 'zombie_moon' }] });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'container recovering' });
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress?.checks).toEqual([expect.objectContaining({ key: 'container', result: 'waiting' })]);
|
||||
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
const receivedAt = await telemetry(0);
|
||||
await processCreations();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment.state).toBe('done');
|
||||
expect(completed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'game', source: 'telemetry', result: 'pass', receivedAt }));
|
||||
expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'verify-map' }));
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps an uncertain failed request immutable while observing and releasing its exact recovery slot', async () => {
|
||||
await register();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'response lost' });
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
await processProvisioning();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'held', operationId: job.deploymentId } });
|
||||
const failureEvents = failed.events.filter(event => event.kind === 'outcome');
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await telemetry(0);
|
||||
await processProvisioning();
|
||||
const recovered = await detail(job.deploymentId);
|
||||
expect(recovered.deployment).toMatchObject({ state: 'failed', finished_at: failed.deployment.finished_at });
|
||||
expect(recovered.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'released', operationId: job.deploymentId } });
|
||||
expect(recovered.events.filter(event => event.kind === 'outcome')).toEqual(failureEvents);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('records occupied and unknown evidence without treating either as zero or resetting eligibility timing', async () => {
|
||||
await register();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null);
|
||||
await processProvisioning();
|
||||
const unknown = await detail(job.deploymentId);
|
||||
expect(unknown.progress?.checks).toEqual([expect.objectContaining({ key: 'occupancy', actual: null, source: null, result: 'unknown' })]);
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await telemetry(3);
|
||||
await processProvisioning();
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phaseStartedAt: unknown.progress?.phaseStartedAt,
|
||||
blocker: { code: 'players-present' }, checks: [{ key: 'occupancy', actual: 3, source: 'telemetry' }] });
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await telemetry(2);
|
||||
await processProvisioning();
|
||||
expect((await detail(job.deploymentId)).progress?.phaseStartedAt).toBe(unknown.progress?.phaseStartedAt);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves failed cleanup evidence through same-ID resume without another destructive command', async () => {
|
||||
await register();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { resumeCleanup } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'cleanup interrupted'); END");
|
||||
const job = destroyServer(server);
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
await processDestructions();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment.state).toBe('failed');
|
||||
expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
|
||||
expect(failed.progress?.checkpoints.some(point => point.key === 'cleanup')).toBe(false);
|
||||
const failure = failed.events.find(event => event.kind === 'outcome')!;
|
||||
db().exec('DROP TRIGGER fail_cleanup');
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
expect(resumeCleanup(job.deploymentId)).toBe(true);
|
||||
await processDestructions();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment).toMatchObject({ id: job.deploymentId, state: 'done', server_exists: 0 });
|
||||
expect(completed.events).toContainEqual(failure);
|
||||
expect(completed.events.some(event => event.kind === 'resumed')).toBe(true);
|
||||
expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'cleanup' }));
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('links the actual removal dependency while later eligible creation bypasses it', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' };
|
||||
const first = queueCreateServer(input), second = queueCreateServer(input);
|
||||
if (!first.ok || !second.ok) throw new Error('Could not queue creation');
|
||||
const removal = beginDeployment('old-instance', 'destroy');
|
||||
db().prepare('INSERT INTO server_creation_dependencies VALUES (?, ?)').run(first.deploymentId, removal);
|
||||
await processCreations();
|
||||
expect((await detail(first.deploymentId)).progress?.blocker).toMatchObject({ code: 'removal-dependency', href: `/provisioning/${removal}` });
|
||||
expect((await detail(first.deploymentId)).deployment.state).toBe('queued');
|
||||
expect((await detail(second.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
});
|
||||
|
||||
it.each(['create', 'delete'] as const)('records uncertainty when the %s reply throws without inventing a successful checkpoint', async command => {
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { queueCreateServer, processCreations, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
let id: string;
|
||||
if (command === 'create') {
|
||||
vi.mocked(komodo.komodoCreateDeployment).mockRejectedValueOnce(new Error('reply lost'));
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
id = job.deploymentId;
|
||||
await processCreations();
|
||||
expect(komodo.komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
} else {
|
||||
await register();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodo.komodoDeleteDeployment).mockRejectedValueOnce(new Error('reply lost'));
|
||||
const job = destroyServer(server);
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
id = job.deploymentId;
|
||||
await processDestructions();
|
||||
expect(komodo.komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
const failed = await detail(id);
|
||||
expect(failed.deployment.state).toBe('failed');
|
||||
expect(failed.progress?.command).toMatchObject({ state: 'uncertain', message: expect.stringContaining('reply lost') });
|
||||
expect(failed.progress?.checkpoints.some(point => point.key === (command === 'create' ? 'create-deployment' : 'delete-deployment'))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['destroy', 'rebuild'] as const)('retains the resumable %s checkpoint when a beta arrives after confirmed remote deletion', async action => {
|
||||
await register();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { resumeCleanup, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodoDeleteDeployment).mockImplementationOnce(async () => {
|
||||
const { script, version } = createScript({ name: 'Late beta', targetRelPath: 'beta.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta') });
|
||||
const beta = db().prepare("INSERT INTO betas (script_id,script_version_id,trigger_kind,state) VALUES (?,?,'manual','running')").run(script.id, version.id).lastInsertRowid;
|
||||
db().prepare("INSERT INTO beta_servers (beta_id,server,had_original,stage) VALUES (?,?,0,'ready')").run(beta, server);
|
||||
return { ok: true };
|
||||
});
|
||||
const job = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
await (action === 'destroy' ? processDestructions() : processProvisioning());
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER });
|
||||
expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
|
||||
expect(action === 'destroy' ? failed.canResumeCleanup : failed.canResumeRebuild).toBe(true);
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back'").run();
|
||||
expect(action === 'destroy' ? resumeCleanup(job.deploymentId) : resumeRebuild(job.deploymentId)).toBe(true);
|
||||
await (action === 'destroy' ? processDestructions() : processProvisioning());
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe(action === 'destroy' ? 'done' : 'queued');
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('ignores occupancy evidence that returns after the request was cancelled', async () => {
|
||||
await register();
|
||||
const { requestLifecycle, cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
expect(cancelProvisioning(job.deploymentId)).toBe(true);
|
||||
return 4;
|
||||
});
|
||||
await processProvisioning();
|
||||
const cancelled = await detail(job.deploymentId);
|
||||
expect(cancelled.deployment.state).toBe('cancelled');
|
||||
expect(cancelled.progress).toMatchObject({ checks: [], blocker: null });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not attach replacement health to a deleted creation after a delayed probe', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await register();
|
||||
return { ok: true, state: 'running' };
|
||||
});
|
||||
await processCreations();
|
||||
const old = await detail(job.deploymentId);
|
||||
expect(old.deployment.server_deleted_at).toBeTruthy();
|
||||
expect(old.progress?.checks).toEqual([]);
|
||||
expect(old.progress?.checkpoints.some(point => point.key.startsWith('verify-'))).toBe(false);
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -24,9 +24,9 @@ describe('provisioning reads with retained lifecycle history', () => {
|
||||
const now = Date.parse('2026-10-05T12:00:00Z');
|
||||
const iso = (offset: number) => new Date(now + offset).toISOString();
|
||||
handle.transaction(() => {
|
||||
const server = handle.prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)');
|
||||
for (let i = 0; i < 16; i++) server.run(`server-${i}`, `server-${i}`);
|
||||
server.run('empty', 'empty');
|
||||
const server = handle.prepare('INSERT INTO servers(name) VALUES (?)');
|
||||
for (let i = 0; i < 16; i++) server.run(`server-${i}`);
|
||||
server.run('empty');
|
||||
const insert = handle.prepare(`INSERT INTO server_deployments
|
||||
(id, server, action, state, stage, started_at, finished_at, submitted_at, server_deleted_at, triggered_by)
|
||||
VALUES (?, ?, ?, 'done', 'Complete', ?, ?, ?, ?, 'test')`);
|
||||
@@ -121,7 +121,7 @@ describe('provisioning reads with retained lifecycle history', () => {
|
||||
legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, table => table
|
||||
.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
|
||||
.replace(' server_deleted_at TEXT,\n', '')));
|
||||
legacy.exec(`INSERT INTO servers(name,komodo_name) VALUES('existing','existing');
|
||||
legacy.exec(`INSERT INTO servers(name) VALUES('existing');
|
||||
INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
|
||||
VALUES('old','existing','create','done','Complete','2025-01-01','fixture');
|
||||
INSERT INTO server_deployment_events(deployment_id,created_at,message)
|
||||
|
||||
@@ -72,7 +72,7 @@ describe('provisioning outcome and evidence presentation', () => {
|
||||
it('shows passed container and telemetry game checks while waiting on the engine map', () => {
|
||||
const detail = fixture();
|
||||
detail.progress = progress({ checks: [
|
||||
{ key: 'container', label: 'Container readiness', result: 'pass', source: 'komodo', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null },
|
||||
{ key: 'container', label: 'Container readiness', result: 'pass', source: 'agent', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null },
|
||||
{ key: 'game', label: 'Game status', result: 'pass', source: 'telemetry', checkedAt: at, receivedAt: at, expected: null, actual: null, message: 'Fresh telemetry accepted' },
|
||||
{ key: 'map', label: 'Engine map', result: 'waiting', source: 'telemetry', checkedAt: at, receivedAt: at, expected: 'zombie_theater', actual: 'zombie_moon', message: null }
|
||||
] });
|
||||
@@ -131,14 +131,13 @@ describe('provisioning outcome and evidence presentation', () => {
|
||||
expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')?.state).toBe('done');
|
||||
});
|
||||
|
||||
it('keeps removal checkpoints separate from replacement creation', () => {
|
||||
const detail = fixture(); detail.deployment.action = 'rebuild'; detail.deployment.state = 'waiting';
|
||||
detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'delete-deployment', label: 'Remote deployment deleted', at, phase: 'delete-deployment' }] });
|
||||
it('keeps confirmed agent removal separate from registry cleanup', () => {
|
||||
const detail = fixture(); detail.deployment.action = 'destroy'; detail.deployment.state = 'waiting';
|
||||
detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'container-action', label: 'Agent confirmed removal', at, phase: 'container-action' }] });
|
||||
const groups = provisioningGroups(detail);
|
||||
expect(groups.map(group => group.key)).toEqual(['request', 'remove', 'create', 'verify']);
|
||||
expect(groups.flatMap(group => group.steps).find(step => step.key === 'delete-deployment')?.state).toBe('done');
|
||||
expect(groups.map(group => group.key)).toEqual(['request', 'command', 'cleanup']);
|
||||
expect(groups.flatMap(group => group.steps).find(step => step.key === 'container-action')?.state).toBe('done');
|
||||
expect(groups.flatMap(group => group.steps).find(step => step.key === 'cleanup')?.state).toBe('waiting');
|
||||
expect(groups.flatMap(group => group.steps).find(step => step.key === 'prepare')?.state).toBe('pending');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -43,10 +43,10 @@ describe('provisioning evidence persistence', () => {
|
||||
const { handle, recordProgress, read } = await setup();
|
||||
recordProgress('operation', { phase: 'verify-game', command: { state: 'uncertain', message: 'Reply was lost' },
|
||||
checkpoint: { key: 'container-action', label: 'Submission attempted', phase: 'container-action' },
|
||||
checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'pass' },
|
||||
checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'pass' },
|
||||
{ key: 'game', label: 'Game', source: 'telemetry', result: 'pass' }] });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'unknown' }] });
|
||||
recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'unknown' }] });
|
||||
expect(read()?.checks).toHaveLength(1);
|
||||
expect(read()?.checks[0].result).toBe('unknown');
|
||||
expect(read()?.checkpoints).toHaveLength(1);
|
||||
@@ -87,7 +87,7 @@ describe('provisioning evidence persistence', () => {
|
||||
const { handle, recordProgress, read } = await setup();
|
||||
expect(() => handle.transaction(() => {
|
||||
handle.prepare("UPDATE server_deployments SET state = 'running' WHERE id = 'operation'").run();
|
||||
recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'delete-deployment', label: 'Remote deletion confirmed' } });
|
||||
recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Remote deletion confirmed' } });
|
||||
throw new Error('Interrupted transaction');
|
||||
})()).toThrow('Interrupted transaction');
|
||||
expect(read()).toBeNull();
|
||||
|
||||
@@ -1,561 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import { readFile, writeFile, access } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_2';
|
||||
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
// Exercise the upgrade path, with history and references present before migration.
|
||||
const old = new Database(fixture.dbPath);
|
||||
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
|
||||
.replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'")
|
||||
.replace("server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server TEXT NOT NULL REFERENCES servers(name),'));
|
||||
old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
|
||||
old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
|
||||
old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
|
||||
old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run();
|
||||
old.close();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
vi.mocked(kick).mockResolvedValue(undefined);
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function operation(id: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
async function queue(action: 'start' | 'restart' | 'stop' | 'rebuild') {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const result = requestLifecycle(server, action);
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
return result.deploymentId;
|
||||
}
|
||||
|
||||
describe('provisioning operations', () => {
|
||||
it('migrates the action constraint without losing row ordering, activity, or foreign keys', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await queue('start');
|
||||
closeDb();
|
||||
expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
|
||||
expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
|
||||
expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
|
||||
expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
});
|
||||
|
||||
it('starts without an RCON empty check and verifies recovery after the grace period', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('start');
|
||||
await processLifecycle();
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledWith(server, 'start');
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ action: 'start', state: 'awaiting-health' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deduplicates requests, rejects conflicting actions, and waits for a fresh empty reading', async () => {
|
||||
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('restart');
|
||||
expect(requestLifecycle(server, 'restart')).toEqual({ ok: true, deploymentId: id });
|
||||
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
|
||||
expect(destroyServer(server)).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
for (const players of [4, null]) {
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting' });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
}
|
||||
await processLifecycle();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not repeat an uncertain restart and retains its recovery slot', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
const id = await queue('restart');
|
||||
await processLifecycle(); await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed', error: 'komodo: timeout' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
});
|
||||
|
||||
it('keeps stop pending on unknown health and completes only on a confirmed stopped state', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { komodoInspectHealthy, komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const id = await queue('stop');
|
||||
await processLifecycle();
|
||||
closeDb();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'network error' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
||||
await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rebuilds from nothing in the same slot and port, then verifies the fresh instance', async () => {
|
||||
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { processCreations, nextLobbyNumber, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const sentinel = join(fixture.t5, server, 'old-instance.dat');
|
||||
await writeFile(sentinel, 'must not survive');
|
||||
const id = await queue('rebuild');
|
||||
// Promotions after queueing must be included in the fresh instance.
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { script } = createScript({ name: 'Fresh promoted script', targetRelPath: 'fresh.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('fresh content') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processLifecycle();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledWith(server);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledWith(server);
|
||||
await expect(access(sentinel)).rejects.toThrow();
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
expect(await operation(id)).toMatchObject({ state: 'queued', action: 'rebuild' });
|
||||
expect(nextAvailablePort()).toBe(28961);
|
||||
expect(nextLobbyNumber('t5_zombie_theater', ['t5_zombie_theater_1'])).toBe(3);
|
||||
expect(requestLifecycle(server, 'rebuild')).toEqual({ ok: true, deploymentId: id });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledWith(expect.objectContaining({ name: server, ports: ['28960:28960/udp', '28960:28960/tcp'] }));
|
||||
await expect(access(sentinel)).rejects.toThrow();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
await processCreations();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === server)).toMatchObject({ revision: 5354, update_status: 'current' });
|
||||
});
|
||||
|
||||
it('rejects a rebuild without a saved key before touching the existing instance', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
removeServerKey(server);
|
||||
expect(requestLifecycle(server, 'rebuild')).toMatchObject({ ok: false, error: expect.stringContaining('Save a server key') });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains the rebuild cleanup checkpoint and reservation while waiting for capacity', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { beginDeployment, deploymentStage, finishDeployment, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('rebuild');
|
||||
// Model a crash after confirmed remote deletion and registry cleanup.
|
||||
deploymentStage(id, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
finishDeployment(id, { ok: false, message: 'cleanup interrupted' });
|
||||
expect(nextAvailablePort()).toBe(28961);
|
||||
expect(resumeRebuild(id)).toBe(true);
|
||||
const occupied = beginDeployment('another-server', 'create');
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
|
||||
closeDb();
|
||||
finishDeployment(occupied, { ok: true });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'queued' });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows explicit destruction of a stalled startup and records the interrupted verification', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('start');
|
||||
await processLifecycle();
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processDestructions();
|
||||
expect(await operation(id)).toMatchObject({ state: 'cancelled', error: 'Server deleted', server_deleted_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('validates lifecycle API input and returns the durable operation ID', async () => {
|
||||
const { POST, GET } = await import('../../src/routes/api/provisioning/+server.js');
|
||||
const submit = (body: unknown) => POST({ request: new Request('http://localhost/api/provisioning', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
|
||||
}) });
|
||||
for (const body of [null, { server, action: 'erase' }, { server: 5, action: 'start' }]) expect((await submit(body)).status).toBe(400);
|
||||
const result = await submit({ server, action: 'start' });
|
||||
expect(result.status).toBe(202);
|
||||
const { deploymentId } = await result.json();
|
||||
expect((await GET({ url: new URL('http://localhost/api/provisioning') }).json()).deployments).toContainEqual(expect.objectContaining({ id: deploymentId, action: 'start', state: 'waiting' }));
|
||||
});
|
||||
|
||||
it('does not create a replacement when teardown fails or resume verification of the old instance', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoDestroyDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
const id = await queue('rebuild');
|
||||
await processLifecycle(); await processCreations(); await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed' });
|
||||
expect(getDeployment(id)?.canResumeVerification).toBe(false);
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
async function incident(id = 'incident') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
// These lifecycle tests exercise an actual container failure. Healthy
|
||||
// containers with fresh game evidence now cancel obsolete health work.
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' });
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
||||
VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
|
||||
return id;
|
||||
}
|
||||
|
||||
async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { gameEvent } = await import('../helpers/telemetry.js');
|
||||
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
||||
ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`)
|
||||
.run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp }));
|
||||
}
|
||||
|
||||
describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
it('keeps the original running intent when a queued removal is cancelled', async () => {
|
||||
const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const removal = destroyServer(server);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal');
|
||||
expect(cancelRemoval(removal.deploymentId)).toBe(true);
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('preserves stop intent across process restarts, including an uncertain stop request', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const id = await queue('stop');
|
||||
finishDeployment(id, { ok: false, message: 'lost response' });
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
// Exercise a legacy upgrade where the stop exists but intent was not stored.
|
||||
db().prepare("UPDATE servers SET desired_state = 'running'").run();
|
||||
db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run();
|
||||
closeDb();
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
await incident();
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' });
|
||||
await queue('start');
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => {
|
||||
const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
|
||||
expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server,
|
||||
label: 'Server down', href: '/maintenance/incident' }]);
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null });
|
||||
await incident(); await report(0);
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
await report(0);
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
|
||||
it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => {
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('restart');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; });
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 });
|
||||
await report(0);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 });
|
||||
});
|
||||
|
||||
it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => {
|
||||
const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
for (const offset of [-90_001, 1]) {
|
||||
await report(0, new Date(Date.now() + offset).toISOString());
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
expect(await probePlayerCount(server)).toBeNull();
|
||||
}
|
||||
await report(0);
|
||||
expect(telemetryEvidence(server, new Date().toISOString())).toBeNull();
|
||||
db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server);
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
});
|
||||
|
||||
it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning();
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild);
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
closeDb();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
removeServerKey(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId });
|
||||
saveServerKey(server, 'saved-key');
|
||||
db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 });
|
||||
});
|
||||
|
||||
it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
let reportHealthy!: (value: { ok: true; state: string }) => void;
|
||||
vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; }));
|
||||
const observation = refreshRestartRollout();
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
reportHealthy({ ok: true, state: 'running' });
|
||||
await observation;
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
});
|
||||
|
||||
it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => {
|
||||
const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
deploymentStage(rebuild.deploymentId, REMOVING_SERVER);
|
||||
finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' });
|
||||
db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run();
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
expect(resumeRebuild(rebuild.deploymentId)).toBe(true);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)")
|
||||
.run(rebuild.deploymentId, new Date().toISOString());
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run();
|
||||
const retry = requestWatchdogLifecycle(server, action, 'incident');
|
||||
if (!retry.ok) throw new Error(retry.error);
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: retry.deploymentId, healthy_at: null });
|
||||
await refreshRestartRollout();
|
||||
// Old process health cannot release capacity before this retry submits.
|
||||
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1);
|
||||
});
|
||||
|
||||
it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
const now = new Date().toISOString();
|
||||
db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now);
|
||||
db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId);
|
||||
db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server);
|
||||
db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.setSystemTime(Date.now() + 15_000); await report(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('cancelling submitted legacy commands', () => {
|
||||
it.each(['start', 'restart', 'stop'] as const)('ignores a late %s response after cancellation', async action => {
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: (value: { ok: true }) => void;
|
||||
const command = action === 'restart' ? komodo.komodoRestart : komodo.komodoRuntimeAction;
|
||||
vi.mocked(command).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const id = await queue(action);
|
||||
const pending = processProvisioning();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
release({ ok: true });
|
||||
await pending;
|
||||
expect(await operation(id)).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(id)).toEqual([]);
|
||||
const replacement = await queue(action);
|
||||
await processProvisioning();
|
||||
expect(await operation(replacement)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(command).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { createSocket, type Socket } from 'node:dgram';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js';
|
||||
import { rconConfigured } from '../../src/lib/server/config.js';
|
||||
import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js';
|
||||
import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js';
|
||||
import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js';
|
||||
@@ -10,11 +10,11 @@ import { nativeInstance, nativeRequest } from '../../src/lib/server/backends/nat
|
||||
vi.mock('node:dgram', () => ({ createSocket: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/config.js', () => ({
|
||||
config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } },
|
||||
komodoConfigured: vi.fn(), rconConfigured: vi.fn()
|
||||
rconConfigured: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() }));
|
||||
|
||||
vi.mock('../../src/lib/server/backends/native.js', () => ({ nativeInstance: vi.fn(), nativeRequest: vi.fn() }));
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
@@ -34,7 +34,6 @@ beforeEach(() => {
|
||||
server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow;
|
||||
vi.mocked(getServer).mockImplementation(() => ({ ...server }));
|
||||
vi.mocked(latestGameSubmission).mockReturnValue(null);
|
||||
vi.mocked(komodoConfigured).mockReturnValue(false);
|
||||
vi.mocked(rconConfigured).mockReturnValue(true);
|
||||
vi.mocked(createSocket).mockImplementation(() => {
|
||||
const socket = new FakeSocket();
|
||||
|
||||
@@ -1,494 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
|
||||
let fixture: Fixture;
|
||||
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
|
||||
const healthyRcon = {
|
||||
ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
|
||||
fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test');
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
|
||||
|
||||
it('ignores a late restart reply after deletion without changing a replacement recovery slot', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { restartInRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let complete!: (value: { ok: true }) => void;
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));
|
||||
const oldOperation = beginDeployment(servers[0], 'restart');
|
||||
const pending = restartInRollout(servers[0], { operationId: oldOperation, action: 'restart' });
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
||||
await seedServerRegistry();
|
||||
const replacement = beginDeployment(servers[0], 'restart');
|
||||
const requestedAt = new Date().toISOString();
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
|
||||
.run(servers[0], requestedAt, replacement);
|
||||
nextTick();
|
||||
complete({ ok: true });
|
||||
expect(await pending).toMatchObject({ kind: 'failed' });
|
||||
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE id = ?').get(oldOperation))
|
||||
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
||||
expect(db().prepare('SELECT operation_id, requested_at, error FROM restart_rollout WHERE server = ?').get(servers[0]))
|
||||
.toEqual({ operation_id: replacement, requested_at: requestedAt, error: null });
|
||||
});
|
||||
|
||||
async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
|
||||
.run(id, action, new Date().toISOString());
|
||||
for (const server of targets) {
|
||||
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
||||
VALUES (?, ?, 1, 'pending')`).run(id, server);
|
||||
}
|
||||
}
|
||||
|
||||
async function rows(id = 'dep-1') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
|
||||
.all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
|
||||
}
|
||||
|
||||
async function beta(targets: string[], waitForHealth = true) {
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
|
||||
return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
|
||||
servers: targets, waitForHealth, waitForMatch: false });
|
||||
}
|
||||
|
||||
describe('rolling deployment restarts', () => {
|
||||
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect((await sweep()).restarted).toBe(1);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
|
||||
await sweep(); // Repeated kick can't bypass the grace period.
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
||||
expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
|
||||
expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
|
||||
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
|
||||
.toMatchObject({ finished_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
|
||||
await deployment('z-oldest', [servers[2]], 'rollback');
|
||||
nextTick();
|
||||
await deployment('a-next', [servers[0]], 'drift');
|
||||
nextTick();
|
||||
await deployment('b-latest', [servers[1]], 'promote');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
|
||||
});
|
||||
|
||||
it('skips occupied and unreachable servers without consuming capacity', async () => {
|
||||
await deployment();
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
});
|
||||
|
||||
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep();
|
||||
nextTick();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let finish!: () => void;
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(() => {
|
||||
resolve();
|
||||
return new Promise((done) => { finish = () => done({ ok: true }); });
|
||||
});
|
||||
});
|
||||
const first = sweep();
|
||||
await entered;
|
||||
const second = sweep();
|
||||
expect(second).toBe(first);
|
||||
// Even a long-running restart request cannot be freed by a health poll.
|
||||
nextTick();
|
||||
const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
|
||||
await refreshRestartRollout();
|
||||
expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
|
||||
finish();
|
||||
await Promise.all([first, second]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
await deployment('dep-2', [servers[0], servers[1]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
await deployment('dep-2', [servers[0]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
||||
expect((await rows('dep-2'))[0].restart_state).toBe('pending');
|
||||
// Provisioning can submit the next job in the pass that verifies the
|
||||
// first, but it must issue a distinct restart and await its recovery.
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
});
|
||||
|
||||
it('preserves occupied slots across app/module restarts', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('adopts pre-upgrade restarts instead of starting a second server', async () => {
|
||||
await deployment();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), servers[0]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
||||
await sweep();
|
||||
expect((await rows())[0].restart_state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not restart a server whose file deployment failed', async () => {
|
||||
await deployment();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
||||
.run(servers[0]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
||||
expect((await rows())[0].error).toBe('write failed');
|
||||
});
|
||||
|
||||
it('releases slots for definite local failures that could not have started a restart', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
||||
await sweep();
|
||||
expect((await rows())[0].restart_state).toBe('failed');
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('beta and deployment coordination', () => {
|
||||
it('limits beta restarts and releases capacity only after recovery', async () => {
|
||||
await beta(servers);
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = betaTick();
|
||||
expect(betaTick()).toBe(first);
|
||||
await first;
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await betaTick();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
||||
await beta([servers[1]]);
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
||||
const { betaId } = await beta([servers[0]], false);
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await betaTick();
|
||||
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
|
||||
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
|
||||
await deployment('dep-1', [servers[1]]);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('checks player occupancy again when resuming a legacy restarting beta', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
||||
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
||||
await betaTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provisioning ownership', () => {
|
||||
it('persists caller requests before probes and completes them without another caller tick', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep(); await sweep();
|
||||
const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
|
||||
const id = source.provisioning_id;
|
||||
expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
|
||||
expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0],
|
||||
label: 'Promotion campaign', href: '/campaigns/campaign' }]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 });
|
||||
await processProvisioning(); nextTick(); await processProvisioning();
|
||||
expect(provisioningOperation(id)?.state).toBe('done');
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('pending');
|
||||
await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('lets concurrent callers submit while only the owner admits restart capacity', async () => {
|
||||
await deployment('campaign', [servers[0]]); await beta([servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
const first = processProvisioning();
|
||||
expect(processProvisioning()).toBe(first);
|
||||
await first;
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not replay an ambiguous campaign restart after recovery or app restart', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb(); vi.resetModules();
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
nextTick(); await fresh.sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('holds a paused beta request and cancels closed work without a beta runner tick', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' });
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
expect(requestLifecycle(servers[0], 'restart').ok).toBe(true);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
});
|
||||
|
||||
it('does not credit a legacy submission with an older completed recovery slot', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const old = new Date().toISOString(); nextTick();
|
||||
const submitted = new Date().toISOString();
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old);
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null });
|
||||
nextTick(); await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string };
|
||||
closeDb(); vi.resetModules();
|
||||
const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const source = { kind: 'beta' as const, id: String(betaId) };
|
||||
expect(requestRestart(servers[0], source).id).toBe(previous.id);
|
||||
releaseRestartRequest(servers[0], source);
|
||||
const next = requestRestart(servers[0], source);
|
||||
expect(next.id).not.toBe(previous.id);
|
||||
expect(provisioningOperation(previous.id)?.state).toBe('cancelled');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(provisioningOperation(next.id)?.state).toBe('awaiting-health');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout configuration', () => {
|
||||
it('honors a configured limit larger than one', async () => {
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
|
||||
await deployment();
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('holds new restarts after lowering a live limit without releasing active slots', async () => {
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 });
|
||||
await deployment();
|
||||
await sweep();
|
||||
saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all()).toHaveLength(2);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
@@ -98,7 +98,7 @@ describe('deferred promotion rollback', () => {
|
||||
expect(existsSync(join(dirname(path), 'never.gsc'))).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps another campaign eligible when withdrawing a shared activation source', async () => {
|
||||
it('keeps another campaign eligible when withdrawing an independent activation source', async () => {
|
||||
const { path, before } = await promotion('script', true);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
@@ -109,7 +109,7 @@ describe('deferred promotion rollback', () => {
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: 'other', server, kind: 'script', relPath: 'other.gsc',
|
||||
content: Buffer.from('other campaign'), inventorySource: 'drift:other' });
|
||||
const shared = requestRestart(server, { kind: 'campaign', id: 'promote' });
|
||||
expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).toBe(shared.id);
|
||||
expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).not.toBe(shared.id);
|
||||
await rollbackPromoteDeployment('promote');
|
||||
await applyCampaign('other');
|
||||
expect(await readFile(path)).toEqual(before);
|
||||
|
||||
@@ -9,7 +9,7 @@ let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']);
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); fixture = await newFixture();
|
||||
delete process.env.KOMODO_ADDRESS; delete process.env.RCON_PASSWORD;
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
function request(content: string, reuse = false) {
|
||||
|
||||
@@ -94,7 +94,7 @@ describe('server chat API', () => {
|
||||
|
||||
it('rejects registered names that escape the server directory', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run('..', '..');
|
||||
db().prepare('INSERT INTO servers (name) VALUES (?)').run('..');
|
||||
expect((await get('..')).status).toBe(400);
|
||||
});
|
||||
|
||||
@@ -130,8 +130,8 @@ describe('server chat API', () => {
|
||||
await report('heartbeat', 40, 8);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(SERVER);
|
||||
db().prepare(`INSERT INTO servers (name, komodo_name, game, map, created_at)
|
||||
VALUES (?, ?, 't5', 'zombie_theater', ?)`).run(SERVER, SERVER, at(70));
|
||||
db().prepare(`INSERT INTO servers (name, game, map, created_at)
|
||||
VALUES (?, 't5', 'zombie_theater', ?)`).run(SERVER, at(70));
|
||||
await report('game_start', 80);
|
||||
await report('heartbeat', 90, 2);
|
||||
await log(['60:50 say;abc;0;Player;previous server', '61:40 say;abc;0;Player;current server'], 100);
|
||||
|
||||
@@ -9,7 +9,7 @@ describe('server detail data', () => {
|
||||
it('filters history before the global limit and includes the matching campaign members', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.exec("INSERT INTO servers (name, komodo_name) VALUES ('target', 'target'), ('other', 'other')");
|
||||
handle.exec("INSERT INTO servers (name) VALUES ('target'), ('other')");
|
||||
handle.transaction(() => {
|
||||
const deployment = handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES (?, 'promote', ?)");
|
||||
const member = handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'done')");
|
||||
@@ -32,7 +32,7 @@ describe('server detail data', () => {
|
||||
});
|
||||
it('identifies plugins, scripts, composed configs and unknown inventory files', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().exec("INSERT INTO servers (name, komodo_name) VALUES ('target', 'target')");
|
||||
db().exec("INSERT INTO servers (name) VALUES ('target')");
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { version } = createScript({ name:'Plugin', kind:'plugin', targetRelPath:'native.bin', content:Buffer.from('plugin') });
|
||||
|
||||
@@ -20,7 +20,7 @@ function database(legacy = false) {
|
||||
}
|
||||
|
||||
function seedHistory(db: Database.Database) {
|
||||
db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'server', 28960);
|
||||
db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960);
|
||||
INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'script', 'test.gsc');
|
||||
INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 4, X'74657374');
|
||||
INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'promoted');
|
||||
@@ -78,7 +78,7 @@ describe('durable server history', () => {
|
||||
expect(db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()).toEqual({ restart_state: 'done', restarted_at: 'restarted' });
|
||||
expect(db.prepare('SELECT content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') });
|
||||
expect(db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) });
|
||||
db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'replacement', 28960);
|
||||
db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960);
|
||||
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new');
|
||||
INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('replacement', 'server', 'health', 'open', 'New incident', 'new', 'new');`);
|
||||
migrateServerHistory(db);
|
||||
@@ -134,7 +134,7 @@ describe('durable server history', () => {
|
||||
('superseded', 'removed', 'rebuild', 'failed', 'Removing server files and registry', 'before', 'ui'),
|
||||
('destroyed', 'removed', 'destroy', 'done', 'Removing server files and registry', 'after', 'ui');
|
||||
INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('superseded');
|
||||
INSERT INTO servers (name, komodo_name, destroyed_at) VALUES ('tombstone', 'tombstone', 'old');
|
||||
INSERT INTO servers (name, destroyed_at) VALUES ('tombstone', 'old');
|
||||
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('gone', 'queued', 'old'), ('tombstone', 'done', 'old');
|
||||
INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('old', 'Old collection', 'old', '["tombstone","gone","new"]');`);
|
||||
migrateServerHistory(db);
|
||||
|
||||
@@ -4,9 +4,6 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn()
|
||||
}));
|
||||
@@ -21,12 +18,13 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
|
||||
|
||||
async function setup() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run();
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
db().prepare("INSERT INTO servers (name, last_error) VALUES ('managed', 'timeout:old process')").run();
|
||||
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
||||
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer };
|
||||
return { db: db(), inspect: vi.mocked(nativeHealth), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer };
|
||||
}
|
||||
|
||||
describe('database fleet and container readiness', () => {
|
||||
@@ -152,7 +150,9 @@ describe('database fleet and container readiness', () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const now = Date.parse('2026-10-02T12:00:00Z');
|
||||
vi.setSystemTime(now);
|
||||
for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`);
|
||||
for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name) VALUES (?)').run(`fast-${i}`);
|
||||
const { prepareNativeHost } = await import('../helpers/native-host.js');
|
||||
await prepareNativeHost(['managed', ...Array.from({ length: 9 }, (_, i) => `fast-${i}`)]);
|
||||
let release!: (result: { ok: true; state: string }) => void;
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
@@ -191,7 +191,7 @@ describe('database fleet and container readiness', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each(['native', 'komodo'])('gives each %s host two slots through inspection and RCON, independently of a stalled host', async backend => {
|
||||
it('gives each host two slots through inspection and RCON, independently of a stalled host', async () => {
|
||||
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
db.prepare("DELETE FROM servers WHERE name='managed'").run();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
@@ -200,9 +200,9 @@ describe('database fleet and container readiness', () => {
|
||||
hosts.enrollHost(hosts.createEnrollment(registered.id).token, { protocolVersion: 1 });
|
||||
for (let instance = 0; instance < 4; instance++) {
|
||||
const name = `host-${host}-${instance}`;
|
||||
db.prepare('INSERT INTO servers (name, komodo_name, komodo_server) VALUES (?, ?, ?)')
|
||||
.run(name, name, backend === 'komodo' ? registered.id : null);
|
||||
if (backend === 'native') db.prepare(`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at)
|
||||
db.prepare('INSERT INTO servers (name) VALUES (?)')
|
||||
.run(name);
|
||||
db.prepare(`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES (?, ?, ?, 'library', 'image', ?, ?)`).run(name, name, registered.id, 28960 + instance, new Date().toISOString());
|
||||
}
|
||||
}
|
||||
@@ -254,7 +254,7 @@ describe('database fleet and container readiness', () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
const now = Date.parse('2026-10-02T12:00:00Z');
|
||||
vi.setSystemTime(now);
|
||||
db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run();
|
||||
db.prepare("INSERT INTO servers (name) VALUES ('slow')").run();
|
||||
let release!: (result: { ok: true; state: string }) => void;
|
||||
inspect.mockImplementation(async name => name === 'slow'
|
||||
? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
|
||||
@@ -531,7 +531,7 @@ describe('database fleet and container readiness', () => {
|
||||
expect(listServers()).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => {
|
||||
it('skips startup RCON, clears old errors, and resumes polling only after the host agent reports readiness', async () => {
|
||||
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||
inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' });
|
||||
|
||||
@@ -9,9 +9,6 @@ vi.mock('node:fs', async (importOriginal) => {
|
||||
return { ...original, writeFileSync: vi.fn(original.writeFileSync) };
|
||||
});
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
@@ -32,17 +29,12 @@ beforeEach(async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T15:00:00Z'));
|
||||
fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', 'test');
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({
|
||||
ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
@@ -84,9 +76,13 @@ async function queueChange(source: Source, targets = [servers[0]]) {
|
||||
return { sourceId, inventory, replacement };
|
||||
}
|
||||
|
||||
// File-storage unit tests exercise activation directly. Native provisioning and
|
||||
// safety suites cover admission, host delivery, player gates, and recovery.
|
||||
async function tick(source: Source) {
|
||||
const { sweep, betaTick } = await import('../helpers/provisioning.js');
|
||||
await (source === 'campaign' ? sweep() : betaTick());
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { activateFiles } = await import('../helpers/staged-files.js');
|
||||
const sources = db().prepare('SELECT DISTINCT source_id FROM staged_file_changes WHERE source_kind=? AND state=\'pending\'').all(source) as { source_id: string }[];
|
||||
for (const row of sources) await activateFiles(source, row.source_id);
|
||||
}
|
||||
|
||||
async function expectLiveUnchanged(inventory: Awaited<ReturnType<typeof queueChange>>['inventory'], server = servers[0]) {
|
||||
@@ -95,96 +91,7 @@ async function expectLiveUnchanged(inventory: Awaited<ReturnType<typeof queueCha
|
||||
expect(getServerScript(server, relPath)).toEqual(inventory);
|
||||
}
|
||||
|
||||
const blockers = ['occupied', 'unknown occupancy', 'missing Komodo', 'missing RCON', 'previous restart', 'rollout capacity'] as const;
|
||||
|
||||
describe.each<Source>(['campaign', 'beta'])('%s staged file activation', (source) => {
|
||||
it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
||||
if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' });
|
||||
if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' });
|
||||
if (blocker === 'previous restart' || blocker === 'rollout capacity') {
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)')
|
||||
.run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString());
|
||||
}
|
||||
const { inventory, replacement } = await queueChange(source);
|
||||
await expectLiveUnchanged(inventory);
|
||||
await tick(source);
|
||||
await expectLiveUnchanged(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
|
||||
// The replacement and its inventory must already be installed when
|
||||
// Komodo sees the activation request, after every admission gate passes.
|
||||
Object.assign(config.komodo, { address: 'http://komodo.test' });
|
||||
Object.assign(config.rcon, { password: 'test' });
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
db().prepare('DELETE FROM restart_rollout').run();
|
||||
let bytesAtRestart: Buffer | undefined;
|
||||
let inventoryAtRestart: typeof inventory;
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
bytesAtRestart = await readFile(destination());
|
||||
inventoryAtRestart = getServerScript(servers[0], relPath);
|
||||
return { ok: true };
|
||||
});
|
||||
await tick(source);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect(bytesAtRestart).toEqual(after);
|
||||
expect(inventoryAtRestart).toMatchObject({
|
||||
script_version_id: replacement.id, sha256: replacement.sha256, size: after.length
|
||||
});
|
||||
});
|
||||
|
||||
it('does not write when its provisioning request is cancelled during the occupancy probe', async () => {
|
||||
const { inventory } = await queueChange(source);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'")
|
||||
.get(servers[0]) as { id: string };
|
||||
cancelProvisioning(operation.id);
|
||||
return 0;
|
||||
});
|
||||
await tick(source);
|
||||
await expectLiveUnchanged(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0]))
|
||||
.toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('never applies old staged work to a replacement server created during its occupancy probe', async () => {
|
||||
await queueChange(source);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const replacementBytes = Buffer.from('new server instance');
|
||||
let replacementInventory: ReturnType<typeof getServerScript>;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
||||
await seedServerRegistry();
|
||||
await writeFile(destination(), replacementBytes);
|
||||
recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement');
|
||||
replacementInventory = getServerScript(servers[0], relPath);
|
||||
return 0;
|
||||
});
|
||||
await tick(source);
|
||||
expect(await readFile(destination())).toEqual(replacementBytes);
|
||||
expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0]))
|
||||
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves existing file permissions when activating staged bytes', async () => {
|
||||
await queueChange(source);
|
||||
@@ -204,260 +111,13 @@ describe.each<Source>(['campaign', 'beta'])('%s staged file activation', (source
|
||||
});
|
||||
|
||||
it('keeps the queued payload across an application restart', async () => {
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(2);
|
||||
const { inventory } = await queueChange(source);
|
||||
await tick(source);
|
||||
await expectLiveUnchanged(inventory);
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
await expectLiveUnchanged(inventory);
|
||||
await tick(source);
|
||||
await expectLiveUnchanged(inventory);
|
||||
const freshRcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
await tick(source);
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect(await readFile(destination())).toEqual(after);
|
||||
});
|
||||
|
||||
it('does not restart or retain rollout capacity when installing staged files fails', async () => {
|
||||
const { inventory } = await queueChange(source);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
// A directory appearing at the destination causes a deterministic I/O
|
||||
// failure even when the test process can bypass ordinary file modes.
|
||||
await rm(destination());
|
||||
await mkdir(destination());
|
||||
await tick(source);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
|
||||
expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0]))
|
||||
.toMatchObject({ state: 'failed', error: expect.any(String) });
|
||||
});
|
||||
});
|
||||
|
||||
describe('staged source eligibility', () => {
|
||||
it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => {
|
||||
const { sourceId, inventory } = await queueChange('campaign');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId);
|
||||
await processProvisioning();
|
||||
await expectLiveUnchanged(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not install a beta withdrawn after it requests provisioning', async () => {
|
||||
const { sourceId, inventory } = await queueChange('beta');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await betaTick();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId);
|
||||
await processProvisioning();
|
||||
await expectLiveUnchanged(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('staged drift removals and review guards', () => {
|
||||
it('defers removing a file and its inventory until the empty-server restart', async () => {
|
||||
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await writeFile(destination(), before);
|
||||
recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture');
|
||||
const inventory = getServerScript(servers[0], relPath);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(1);
|
||||
await runDriftDeployment({ servers: [servers[0]] });
|
||||
await expectLiveUnchanged(inventory);
|
||||
await tick('campaign');
|
||||
await expectLiveUnchanged(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
let fileExistedAtRestart: boolean | undefined;
|
||||
let inventoryAtRestart: typeof inventory;
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
|
||||
fileExistedAtRestart = existsSync(destination());
|
||||
inventoryAtRestart = getServerScript(servers[0], relPath);
|
||||
return { ok: true };
|
||||
});
|
||||
await tick('campaign');
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect(fileExistedAtRestart).toBe(false);
|
||||
expect(inventoryAtRestart).toBeUndefined();
|
||||
});
|
||||
|
||||
it('fails without overwriting a reviewed target changed while its restart was queued', async () => {
|
||||
const { inventory } = await queueChange('campaign');
|
||||
const externalChange = Buffer.from('changed outside Electric Cherry while queued');
|
||||
await writeFile(destination(), externalChange);
|
||||
await tick('campaign');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(await readFile(destination())).toEqual(externalChange);
|
||||
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0]))
|
||||
.toMatchObject({ error: expect.stringContaining('changed while queued') });
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('shared activation and partial writes', () => {
|
||||
it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => {
|
||||
const first = await queueChange('campaign');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const second = await runDriftDeployment({ servers: [servers[0]] });
|
||||
await sweep();
|
||||
const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[];
|
||||
expect(linked).toHaveLength(2);
|
||||
expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1);
|
||||
db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId);
|
||||
await processProvisioning();
|
||||
await expectLiveUnchanged(first.inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' });
|
||||
db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId);
|
||||
await tick('campaign');
|
||||
expect(await readFile(destination())).toEqual(after);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
|
||||
.toEqual([{ state: 'applied' }, { state: 'applied' }]);
|
||||
});
|
||||
|
||||
it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => {
|
||||
const campaign = await queueChange('campaign');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
await sweep();
|
||||
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes'));
|
||||
await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }],
|
||||
servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
|
||||
await expectLiveUnchanged(campaign.inventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT * FROM betas').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId))
|
||||
.toEqual({ restart_state: 'pending' });
|
||||
expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId))
|
||||
.toEqual({ state: 'pending' });
|
||||
});
|
||||
|
||||
it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => {
|
||||
const beta = await queueChange('beta');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
// Recreate overlapping work retained from before the admission guard.
|
||||
// The beta was already open; this older drift still expects its original
|
||||
// live bytes and must fail independently of the later restoration.
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)")
|
||||
.run(new Date(Date.now() - 60_000).toISOString());
|
||||
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
||||
VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]);
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script',
|
||||
relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift',
|
||||
expectedBeforeSha: sha256Buffer(before) });
|
||||
await sweep();
|
||||
await tick('beta');
|
||||
expect(await readFile(destination())).toEqual(after);
|
||||
const rollback = await rollbackBeta(Number(beta.sourceId));
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await tick('campaign');
|
||||
expect(await readFile(destination())).toEqual(before);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?')
|
||||
.get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string };
|
||||
const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?')
|
||||
.get(rollback.deploymentId) as { provisioning_id: string };
|
||||
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('Superseded by rollback') });
|
||||
expect(restoration.provisioning_id).not.toBe(old.provisioning_id);
|
||||
});
|
||||
|
||||
it('records partial installation without restarting and never automatically retries the failed write', async () => {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after });
|
||||
const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after });
|
||||
const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc');
|
||||
for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) {
|
||||
await writeFile(path, before);
|
||||
recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture');
|
||||
}
|
||||
const secondInventory = getServerScript(servers[0], 'second.gsc');
|
||||
await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })),
|
||||
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
|
||||
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
|
||||
vi.mocked(writeFileSync).mockImplementation((path, content, options) => {
|
||||
if (path === secondPath) throw new Error('Simulated disk write failure');
|
||||
actualFs.writeFileSync(path, content, options);
|
||||
});
|
||||
await tick('beta');
|
||||
expect(await readFile(destination())).toEqual(after);
|
||||
expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id);
|
||||
expect(await readFile(secondPath)).toEqual(before);
|
||||
expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
|
||||
.toEqual([{ state: 'applied' }, { state: 'applying' }]);
|
||||
expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get())
|
||||
.toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
||||
const attempts = vi.mocked(writeFileSync).mock.calls.length;
|
||||
await tick('beta');
|
||||
expect(writeFileSync).toHaveBeenCalledTimes(attempts);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollback supersedes provisioning', () => {
|
||||
it.each(['when-empty', 'manual'] as const)('replaces an unhealthy beta restart with a %s rollback without waiting for recovery', async restart => {
|
||||
const beta = await queueChange('beta');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await tick('beta');
|
||||
const old = db().prepare('SELECT provisioning_id FROM beta_servers WHERE beta_id=?').get(beta.sourceId) as { provisioning_id: string };
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'container failed to start' });
|
||||
const rollback = await rollbackBeta(Number(beta.sourceId), { restart });
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(old.provisioning_id)).toEqual([]);
|
||||
const replacement = db().prepare('SELECT provisioning_id,restart_state FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as { provisioning_id: string; restart_state: string };
|
||||
expect(replacement.provisioning_id).not.toBe(old.provisioning_id);
|
||||
await tick('campaign');
|
||||
expect(await readFile(destination())).toEqual(restart === 'manual' ? after : before);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(restart === 'manual' ? 1 : 2);
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(replacement.provisioning_id))
|
||||
.toEqual({ state: restart === 'manual' ? 'waiting' : 'awaiting-health' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@ beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']);
|
||||
// Provisioning must retain queued work while integrations are unset.
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
|
||||
@@ -95,7 +95,7 @@ describe('sweep', () => {
|
||||
for (const r of rows) expect(r.restart_state).toBe('pending');
|
||||
const operations = db().prepare('SELECT state, stage FROM server_deployments').all();
|
||||
expect(operations).toHaveLength(2);
|
||||
for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting for Komodo and RCON configuration' });
|
||||
for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting to verify the server is empty' });
|
||||
});
|
||||
|
||||
it('is a no-op when nothing is pending', async () => {
|
||||
|
||||
@@ -5,46 +5,33 @@ let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('KOMODO_ADDRESS', '');
|
||||
vi.stubEnv('KOMODO_API_KEY', '');
|
||||
vi.stubEnv('KOMODO_API_SECRET', '');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', '');
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
it('directs a fresh controller to native hosts and verified libraries without probing Komodo', async () => {
|
||||
const probe = vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe');
|
||||
it('directs a fresh controller to host enrollment and verified game files', async () => {
|
||||
const { GET } = await import('../../src/routes/api/system-status/+server.js');
|
||||
const response = await GET({ url: new URL('http://localhost/api/system-status') });
|
||||
const response = await GET();
|
||||
const body = await response.json();
|
||||
expect(response.headers.get('cache-control')).toBe('no-store');
|
||||
expect(body.checks.find((check: { id: string }) => check.id === 'hosts').state).toBe('unconfigured');
|
||||
expect(body.checks.find((check: { id: string }) => check.id === 'game-files').state).toBe('attention');
|
||||
expect(body.checks.some((check: { id: string }) => check.id === 'komodo')).toBe(false);
|
||||
expect(body.checks.map((check: { id: string }) => check.id)).toEqual(['hosts', 'runtime', 'game-files', 'rcon', 'keys', 'first-server']);
|
||||
expect(body.checks[0]).toMatchObject({ state: 'unconfigured', href: '/hosts' });
|
||||
expect(body.checks[2].state).toBe('attention');
|
||||
expect(body.readyServers).toBe(0);
|
||||
expect(probe).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(body)).not.toContain('fixture-rcon-secret');
|
||||
});
|
||||
|
||||
it('never exposes credentials or claims that network reachability validates them', async () => {
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'https://komodo.example.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'fixture-private-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'fixture-private-secret');
|
||||
vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret');
|
||||
vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 443, resolvedIp: '192.0.2.1', outcome: 'connected', elapsedMs: 1 });
|
||||
it('reports an enrolled but disconnected host as needing attention without issuing commands', async () => {
|
||||
const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const host = createHost({ name: 'Game host' });
|
||||
const enrolled = enrollHost(createEnrollment(host.id).token, { protocolVersion: 1 });
|
||||
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', host.id);
|
||||
const { GET } = await import('../../src/routes/api/system-status/+server.js');
|
||||
const text = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).text();
|
||||
for (const secret of ['fixture-private-key', 'fixture-private-secret', 'fixture-rcon-secret']) expect(text).not.toContain(secret);
|
||||
const connection = JSON.parse(text).checks.find((check: { id: string }) => check.id === 'komodo');
|
||||
expect(connection.state).toBe('ready');
|
||||
expect(connection.detail).toContain('does not validate API credentials');
|
||||
});
|
||||
|
||||
it('reports a configured unreachable host as needing attention', async () => {
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.example.test:9120');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'key'); vi.stubEnv('KOMODO_API_SECRET', 'secret');
|
||||
vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 9120, resolvedIp: null, outcome: 'dns-fail', elapsedMs: 1 });
|
||||
const { GET } = await import('../../src/routes/api/system-status/+server.js');
|
||||
const body = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).json();
|
||||
expect(body.checks.find((check: { id: string }) => check.id === 'komodo').state).toBe('attention');
|
||||
const response = await GET();
|
||||
const text = await response.text();
|
||||
const body = JSON.parse(text);
|
||||
expect(body.checks[0]).toMatchObject({ state: 'attention', detail: expect.stringContaining('0 connected') });
|
||||
expect(text).not.toContain(enrolled.credential);
|
||||
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -14,7 +14,7 @@ const cfgPath = (server: string) => join(fixture.t6, server, 'configs/dedicated_
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([town, farm, t5]);
|
||||
vi.stubEnv('KOMODO_ADDRESS', '');
|
||||
|
||||
vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
|
||||
@@ -210,7 +210,7 @@ describe('Fleet report clock', () => {
|
||||
const { fleetServers, hydrateFleet } = await setup();
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
emit(epoch + 2000, [status(epoch + 2000, { players: 3 })]);
|
||||
const integrations = { rcon: true, komodo: false, rconTimeoutMs: 3000, defaultRconHost: '' };
|
||||
const integrations = { rcon: true, rconTimeoutMs: 3000, defaultRconHost: '' };
|
||||
hydrateFleet({ serverNow: epoch + 1000, servers: [server(epoch + 1000), server(epoch, { name: 'two' })], operations: [], integrations });
|
||||
expect(get(fleetServers)).toHaveLength(2);
|
||||
expect(get(fleetServers)[0].last_players).toBe(3);
|
||||
|
||||
+26
-362
@@ -1,14 +1,10 @@
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
@@ -20,7 +16,8 @@ let fixture: Fixture;
|
||||
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
|
||||
let watchdog: Engine;
|
||||
let database: typeof import('../../src/lib/server/db/index.js');
|
||||
let komodo: typeof import('../../src/lib/server/integrations/komodo.js');
|
||||
let backend: typeof import('../../src/lib/server/backends/native.js');
|
||||
let refreshHost: () => unknown;
|
||||
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -29,22 +26,18 @@ beforeEach(async () => {
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
for (const [key, value] of Object.entries({ RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1' })) vi.stubEnv(key, value);
|
||||
database = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
||||
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
}
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
refreshHost = (await prepareNativeHost([server])).heartbeat;
|
||||
backend = await import('../../src/lib/server/backends/native.js');
|
||||
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
|
||||
const { mockAgentFiles } = await import('../helpers/agent-files.js');
|
||||
await mockAgentFiles();
|
||||
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
@@ -56,8 +49,8 @@ afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAl
|
||||
const incidents = () => watchdog.watchdogOverview().incidents;
|
||||
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
|
||||
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
|
||||
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
|
||||
function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
||||
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); }
|
||||
function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
||||
async function pollStatus() {
|
||||
const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
|
||||
await worker.fanoutStatus();
|
||||
@@ -78,7 +71,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick();
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
}
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([]);
|
||||
@@ -98,7 +91,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
}
|
||||
expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(3);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(3);
|
||||
expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
|
||||
});
|
||||
|
||||
@@ -113,7 +106,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
advance(60_000);
|
||||
await watchdog.watchdogTick();
|
||||
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(1);
|
||||
advance(90_001);
|
||||
await pollStatus();
|
||||
await watchdog.watchdogTick();
|
||||
@@ -205,7 +198,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
await watchdog.watchdogTick();
|
||||
}
|
||||
expect(scan).toHaveBeenCalledTimes(1);
|
||||
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5);
|
||||
expect(backend.nativeHealth).toHaveBeenCalledTimes(5);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
await pollStatus();
|
||||
await watchdog.watchdogTick();
|
||||
@@ -256,32 +249,11 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps paused recovery observational and resumes one pending incident', async () => {
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
down();
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 4 });
|
||||
expect(operations()).toEqual([]);
|
||||
database.closeDb();
|
||||
expect(watchdog.watchdogOverview().enabled).toBe(false);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not restart an unknown Komodo state even after repeated sustained observations', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo request timed out' });
|
||||
it('does not restart an unknown host agent state even after repeated sustained observations', async () => {
|
||||
vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' });
|
||||
for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
|
||||
expect(operations()).toEqual([]);
|
||||
@@ -297,7 +269,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
it('allows a longer initial startup grace period before requesting recovery', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(operations()).toEqual([]);
|
||||
await watchdog.watchdogTick(true);
|
||||
@@ -346,79 +318,6 @@ describe('Watchdog detection and ownership', () => {
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
});
|
||||
|
||||
it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
|
||||
const row = await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('holds queued health recovery when container readiness becomes unknown', async () => {
|
||||
await sustainedFailure();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const restart = operations()[0];
|
||||
const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
expect(cancelProvisioning(restart.id)).toBe(true);
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return 0;
|
||||
});
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
|
||||
expect.objectContaining({ action: 'stop', state: 'waiting' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
|
||||
.run(server, new Date().toISOString()).lastInsertRowid;
|
||||
const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
|
||||
expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
if (action === 'rebuild') {
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
}
|
||||
const queued = operations().at(-1)!;
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
|
||||
expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
|
||||
await processProvisioning();
|
||||
expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves intentional stop intent after completion and database reopening', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const stop = requestLifecycle(server, 'stop');
|
||||
@@ -430,7 +329,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
|
||||
@@ -447,12 +346,12 @@ describe('Watchdog detection and ownership', () => {
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return { ok: false, state: 'exited', reason: 'Container exited' };
|
||||
});
|
||||
@@ -477,7 +376,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
}
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(backend.nativeHealth).not.toHaveBeenCalled();
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(operations()).toHaveLength(1);
|
||||
|
||||
@@ -495,7 +394,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
|
||||
it('stops probing when provisioning takes ownership during the container inspection', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
|
||||
expect(requestLifecycle(server, 'restart').ok).toBe(true);
|
||||
return { ok: true, state: 'running' };
|
||||
});
|
||||
@@ -507,173 +406,6 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog recovery persistence', () => {
|
||||
it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
});
|
||||
|
||||
it('defers escalation while current container health is unknown', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
|
||||
});
|
||||
|
||||
it('keeps failed submission history when fresh evidence later releases recovery', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ healthy_at: new Date().toISOString() });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
|
||||
});
|
||||
|
||||
it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
|
||||
database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
|
||||
return healthyRcon;
|
||||
});
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
|
||||
expect(operations().some(op => op.action === 'rebuild')).toBe(false);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
});
|
||||
|
||||
it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
const rebuild = operations()[1];
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
removeServerKey(server);
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
const id = onlyIncident().id;
|
||||
expect(onlyIncident().state).toBe('attention');
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
down();
|
||||
await processDestructions();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]);
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'attention', server_deleted_at: expect.any(String) });
|
||||
expect(watchdog.updateWatchdogIncident(id, 'retry')).toMatchObject({ ok: false, status: 409 });
|
||||
});
|
||||
|
||||
it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(operations()[0].state).toBe('awaiting-health');
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS - 1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
advance(1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(2);
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
||||
await processProvisioning();
|
||||
expect(operations()[1].state).toBe('failed');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
database.closeDb();
|
||||
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); }
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not escalate a timed out restart while automatic recovery is paused', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
});
|
||||
|
||||
it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
||||
await processProvisioning();
|
||||
await watchdog.watchdogTick(true);
|
||||
const failed = operations()[1];
|
||||
expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' });
|
||||
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 });
|
||||
const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server);
|
||||
expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null });
|
||||
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockClear();
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(operations()).toHaveLength(2);
|
||||
|
||||
expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(2);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
const retry = operations()[2];
|
||||
expect(retry).toMatchObject({ action: 'restart', state: 'waiting' });
|
||||
expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 });
|
||||
expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3);
|
||||
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect(operations()[2].state).toBe('awaiting-health');
|
||||
});
|
||||
|
||||
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
|
||||
down();
|
||||
@@ -770,41 +502,6 @@ describe('Watchdog APIs', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog drift campaigns', () => {
|
||||
it('queues an audited rebuild without changing files until provisioning executes and retains its campaign through cleanup', async () => {
|
||||
await mkdir(join(fixture.t5, server, 'configs'), { recursive: true });
|
||||
const path = join(fixture.t5, server, 'configs', 'stray.cfg');
|
||||
await writeFile(path, 'unexpected content');
|
||||
await watchdog.watchdogTick(true);
|
||||
const row = onlyIncident();
|
||||
expect(row).toMatchObject({ kind: 'drift', state: 'recovering' });
|
||||
const action = watchdog.watchdogDetail(row.id)!.actions[0];
|
||||
expect(action).toMatchObject({ action: 'rebuild', campaign_id: expect.any(String), operation_state: 'waiting' });
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(database.db().prepare('SELECT rel_path, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(action.campaign_id))
|
||||
.toEqual([{ rel_path: 'stray.cfg', content_before: Buffer.from('unexpected content'), content_after: null }]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
down();
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'queued' })]);
|
||||
await expect(access(path)).rejects.toThrow();
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
database.closeDb();
|
||||
await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id))
|
||||
.toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' });
|
||||
await processCreations();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
advance(1);
|
||||
await processCreations(); await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' });
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'done' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
@@ -812,43 +509,10 @@ describe('Watchdog drift campaigns', () => {
|
||||
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
||||
await writeFile(path, 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('Save a server key') });
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('server key are required') });
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Maintenance recovery policy boundaries', () => {
|
||||
it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => {
|
||||
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
||||
policies.setMaintenancePolicy('health', false);
|
||||
down();
|
||||
for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(operations()).toEqual([]);
|
||||
expect(onlyIncident().kind).toBe('health');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
|
||||
policies.setMaintenancePolicy('drift', false);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
it('global pause blocks escalation without disabling the saved health policy', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
const policies = await import('../../src/lib/server/maintenance/policies.js');
|
||||
policies.pauseMaintenance(true);
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
expect(policies.maintenancePolicies().health).toBe(true);
|
||||
policies.pauseMaintenance(false); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(2);
|
||||
expect(operations()[1].action).toBe('rebuild');
|
||||
expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user