import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; import { defaultPlan, requirement, type BetaPlan } from '../../src/lib/beta-plan.js'; vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() })); // Planner tests observe the provisioning contract. Host command delivery and // disruption safety are exercised by the native provisioning suites. vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ processProvisioning: async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js'); const { probePlayerEvidence } = await import('../../src/lib/server/fleet/status-evidence.js'); const jobs = db() .prepare( "SELECT id,server,state FROM server_deployments WHERE action='restart' AND state IN ('waiting','awaiting-health')" ) .all() as { id: string; server: string; state: string }[]; for (const job of jobs) { if (job.state === 'awaiting-health') { db() .prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id=?") .run(new Date().toISOString(), job.id); } else if ((await probePlayerEvidence(job.server)).players === 0) { applyStagedFilesForOperation(job.id, job.server); db() .prepare( "UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?" ) .run(new Date().toISOString(), job.id); } } } })); let fixture: Fixture; const names = ['t5_zombie_theater_1', 't5_zombie_theater_2']; const now = () => new Date().toISOString(); beforeEach(async () => { vi.resetModules(); vi.resetAllMocks(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-17T15:00:00Z')); fixture = await newFixture(names); for (const name of names) { await mkdir(join(fixture.t5, name, 'logs'), { recursive: true }); await writeFile(join(fixture.t5, name, 'logs/console.log'), 'old script error\n'); await writeFile( join(fixture.t5, name, 'logs/games_sp.log'), 'InitGame:\n 0:00 J;old;1;old\nExitLevel: executed\n' ); } vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 1 }, status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.useRealTimers(); }); async function openPlan(plan: BetaPlan, servers = [names[0]]) { const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script } = createScript({ releaseVersion: '1.0.0', name: 'Beta test', targetRelPath: 'test.gsc', content: Buffer.from('main() {}') }); const result = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }], servers, waitForHealth: true, waitForMatch: false, testPlan: plan }); db().prepare('UPDATE betas SET opened_at = ? WHERE id = ?').run(now(), result.betaId); return result.betaId; } async function tick(seconds = 15, players = 4, healthy = true) { vi.setSystemTime(Date.now() + seconds * 1000); const { db } = await import('../../src/lib/server/db/index.js'); db() .prepare( "UPDATE servers SET last_checked_at = ?, last_seen_at = ?, last_players = ?, last_error = ?, container_state = 'running'" ) .run(now(), now(), players, healthy ? null : 'timeout'); const { betaTick } = await import('../helpers/provisioning.js'); await betaTick(); } const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc'); describe('durable beta test plans', () => { it.each([ { scope: 'quorum', fileScope: 'game', otherServer: 't6_zombie_town_1', otherMap: 'zm_transit' }, { scope: 'game', fileScope: 'map', otherServer: 't5_zombie_cosmodrome_1', otherMap: 'zombie_cosmodrome' } ] as const)( 'requires a passing participant for every file with $scope coverage', async ({ scope, fileScope, otherServer, otherMap }) => { const otherGame = otherServer.slice(0, 2); await mkdir( otherGame === 't6' ? join(fixture.t6, otherServer, 'runtime/plutonium/storage/t6/scripts/zm') : join(fixture.t5, otherServer, 'runtime/plutonium/storage/t5/raw/scripts/sp'), { recursive: true } ); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); const { createScript, getVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, getBeta, listBetaServers, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); const files = [ fileScope === 'game' ? 't5' : 'zombie_theater', fileScope === 'game' ? otherGame : otherMap ].map((scopeValue) => createScript({ releaseVersion: '1.0.0', name: `Test ${scopeValue}`, targetRelPath: `${scopeValue}.gsc`, scopeType: fileScope, scopeValue, content: Buffer.from(`${scopeValue} beta`) }) ); const { betaId } = await openBeta({ scripts: files.map((file) => ({ scriptId: file.script.id, scriptVersionId: file.version.id })), servers: [...names, otherServer], waitForHealth: true, waitForMatch: false, testPlan: { ...defaultPlan(), scope, quorum: 1, requirements: [requirement('population', 'players')] } }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); await tick(); expect( listBetaServers(betaId).find((server) => server.server === names[0])?.stage ).toBe('ready'); expect(getBeta(betaId)?.state).toBe('running'); expect(testView(betaId)?.outcome).toBe('active'); expect(files.map((file) => getVersion(file.version.id)?.state)).toEqual([ 'untested', 'untested' ]); await expect(promoteBeta(betaId)).rejects.toThrow('Test plan has not passed'); await postTelemetry(gameEvent('game_start', 25, 4, otherServer, otherMap)); await tick(); expect(getBeta(betaId)?.state).toBe('ready-promote'); expect(testView(betaId)?.outcome).toBe('passed'); expect(files.map((file) => getVersion(file.version.id)?.state)).toEqual([ 'tested', 'tested' ]); // Coverage still permits replicas to remain unproven once each file // has a passing participant; it does not silently become an all gate. expect( listBetaServers(betaId).find((server) => server.server === names[1])?.stage ).toBe('observing'); } ); it('requires fresh gameplay after a revision and promotes exactly the replacement version', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick(); const { getBeta, listBetaScripts, reviseBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); const { getScript } = await import('../../src/lib/server/fleet/scripts.js'); const file = listBetaScripts(id)[0]; expect(getBeta(id)?.state).toBe('ready-promote'); await reviseBeta(id, 1, [ { scriptId: file.script_id, releaseVersion: '1.0.1', content: Buffer.from('main() { /* fixed */ }') } ]); const replacement = listBetaScripts(id)[0].script_version_id; expect(replacement).not.toBe(file.script_version_id); await postTelemetry(gameEvent('game_end')); // Still running the old revision. expect(testView(id)?.evidence).toHaveLength(0); await postTelemetry(gameEvent('heartbeat', 1, 0)); await tick(); await tick(); expect(getBeta(id)?.state).toBe('running'); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick(); expect(getBeta(id)?.state).toBe('ready-promote'); await promoteBeta(id); expect(getScript(file.script_id)?.current_version_id).toBe(replacement); }); it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => { const id = await openPlan(defaultPlan()); const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js'); await postTelemetry(gameEvent('heartbeat', 4, 2)); await tick(); expect(listBetaServers(id)[0].stage).toBe('awaiting-empty'); await postTelemetry(gameEvent('heartbeat', 4, 0)); await tick(); expect(listBetaServers(id)[0].stage).toBe('awaiting-health'); }); it('holds file deployment until each wave passes, and rollback leaves undeployed files alone', async () => { await writeFile(scriptPath(names[1]), 'original untracked file'); const id = await openPlan({ ...defaultPlan(), waveSize: 1 }, names); expect(await readFile(scriptPath(names[1]), 'utf8')).toBe('original untracked file'); const { listBetaServers, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); expect(listBetaServers(id).map((s) => s.stage)).toEqual(['awaiting-empty', 'queued']); await rollbackBeta(id); expect(await readFile(scriptPath(names[1]), 'utf8')).toBe('original untracked file'); expect(existsSync(scriptPath(names[0]))).toBe(false); }); it('expands only after observed healthy time, then requires the next wave to pass', async () => { const id = await openPlan({ ...defaultPlan(), waveSize: 1 }, names); const { getBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js'); await tick(); await tick(); expect(existsSync(scriptPath(names[1]))).toBe(false); for (let i = 0; i < 4; i++) await tick(); expect(existsSync(scriptPath(names[1]))).toBe(false); expect(getBeta(id)?.state).toBe('running'); expect(listBetaServers(id)[1].stage).toBe('awaiting-empty'); await tick(); expect(existsSync(scriptPath(names[1]))).toBe(true); for (let i = 0; i < 6; i++) await tick(); expect(getBeta(id)?.state).toBe('ready-promote'); }); it('uses script events instead of logs and keeps match progress across app restarts', async () => { await postTelemetry(gameEvent('game_end')); const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] }); await tick(91); await tick(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); await appendFile( join(fixture.t5, names[0], 'logs/games_sp.log'), 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: executed\n' ); await tick(); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('game_start')); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); await postTelemetry(gameEvent('game_end')); await tick(); expect(testView(id)?.servers[0].progress[0].current).toBe(1); expect(testView(id)?.outcome).toBe('passed'); }); it('automatically credits ordinary rounds only to active deployed beta servers', async () => { const id = await openPlan( { ...defaultPlan(), waveSize: 1, requirements: [requirement('round', 'round')] }, names ); await postTelemetry(gameEvent('round_change')); await tick(91); await tick(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('round_change', 99, 4, names[1])); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('round_change')); expect(testView(id)?.servers[0].progress[0].current).toBe(25); expect(testView(id)?.evidence.filter((e) => e.kind === 'round_change')).toHaveLength(1); await tick(); expect(existsSync(scriptPath(names[1]))).toBe(false); await postTelemetry(gameEvent('heartbeat', 99, 0, names[1])); await tick(); expect(existsSync(scriptPath(names[1]))).toBe(true); }); it('resets evidence after planned recovery and consumes the same simple reports afterward', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [ requirement('population', 'players'), requirement('recovery', 'recovery') ] }); await tick(); await tick(); const { currentRun, testView } = await import('../../src/lib/server/fleet/beta-tests.js'); const first = currentRun(id, names[0])!; expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('game_start')); await tick(91); await tick(); expect(currentRun(id, names[0])?.id).not.toBe(first.id); expect(currentRun(id, names[0])?.phase).toBe(1); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('game_start')); await tick(); expect(testView(id)?.outcome).toBe('passed'); }); it('treats missing evidence as inconclusive and refuses promotion', async () => { const id = await openPlan({ ...defaultPlan(), timeoutHours: 0.01, requirements: [requirement('round', 'round')] }); await tick(); await tick(); await tick(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); expect(testView(id)?.outcome).toBe('inconclusive'); await expect(promoteBeta(id)).rejects.toThrow('Test plan has not passed'); }); it('revokes readiness if health fails while awaiting manual promotion', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [ requirement('population', 'players'), requirement('failure-budget', 'outages') ] }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); await tick(); const { getBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); expect(getBeta(id)?.state).toBe('ready-promote'); await tick(15, 0, false); // Fresh occupied telemetry remains healthy despite an RCON timeout. expect(getBeta(id)?.state).toBe('ready-promote'); await tick(61, 0, false); expect(getBeta(id)?.state).toBe('running'); await expect(promoteBeta(id)).rejects.toThrow(); }); it('automatically rolls back only the deployed wave and queues safe rollback restarts', async () => { await writeFile(scriptPath(names[0]), 'original'); const id = await openPlan( { ...defaultPlan(), waveSize: 1, failureAction: 'rollback', requirements: [ requirement('round', 'round'), requirement('failure-budget', 'outages') ] }, names ); await tick(); await tick(); await tick(15, 0, false); const { getBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect(getBeta(id)?.state).toBe('rolled-back'); // Rollback now replaces the unhealthy restart in the same worker pass. expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original'); expect(existsSync(scriptPath(names[1]))).toBe(false); expect(db().prepare('SELECT server, restart_state FROM deployment_servers').all()).toEqual([ { server: names[0], restart_state: 'pending' } ]); const { sweep } = await import('../helpers/provisioning.js'); await sweep(); expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original'); }); it('keeps snapshots and holds after a paused failure, without deploying subsequent waves', async () => { const id = await openPlan( { ...defaultPlan(), waveSize: 1, failureAction: 'pause', requirements: [ requirement('round', 'round'), requirement('failure-budget', 'outages') ] }, names ); await tick(); await tick(); await tick(15, 0, false); await tick(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); expect(testView(id)?.outcome).toBe('paused'); expect(existsSync(scriptPath(names[0]))).toBe(true); expect(existsSync(scriptPath(names[1]))).toBe(false); }); it('retains artifact ownership while ready for promotion', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [requirement('population', 'players')] }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); await tick(); const { heldByBeta, ownerOf } = await import('../../src/lib/server/fleet/servers.js'); expect(heldByBeta(names[0]).get('test.gsc')).toBe(id); expect(ownerOf(names[0], 'test.gsc')).toBe(id); const { promoteBeta, getBeta } = await import('../../src/lib/server/fleet/beta.js'); await promoteBeta(id); expect(getBeta(id)?.state).toBe('promoted'); expect(ownerOf(names[0], 'test.gsc')).toBeNull(); }); it('counts successive games with the same players and ignores repeated game ends', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [{ ...requirement('matches', 'matches'), target: 2 }] }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await postTelemetry(gameEvent('game_end')); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); expect(testView(id)?.servers[0].progress[0].current).toBe(1); await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick(); expect(testView(id)?.servers[0].progress[0].current).toBe(2); expect(testView(id)?.outcome).toBe('passed'); }); it('records checklist sign-off and starts a fresh attempt without changing script payloads', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [requirement('round', 'round'), requirement('manual', 'hud')] }); await tick(); await tick(); const telemetry = await import('../../src/routes/api/telemetry/+server.js'); expect(telemetry).not.toHaveProperty('GET'); expect(await (await postTelemetry(gameEvent('round_change'))).json()).not.toHaveProperty( 'test' ); const controls = await import('../../src/routes/api/betas/[id]/test/+server.js'); const result = await controls.POST({ params: { id: String(id) }, request: new Request('http://test', { method: 'POST', body: JSON.stringify({ action: 'sign-off', server: names[0], requirementId: 'hud', note: 'Josh: HUD verified' }) }) }); expect(result.status).toBe(200); await tick(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); expect(testView(id)?.outcome).toBe('passed'); const retry = await controls.POST({ params: { id: String(id) }, request: new Request('http://test', { method: 'POST', body: JSON.stringify({ action: 'retry' }) }) }); expect(retry.status).toBe(200); await postTelemetry(gameEvent('round_change')); expect(testView(id)?.servers).toHaveLength(0); await tick(91); await tick(); expect(testView(id)?.servers[0].progress[0].current).toBe(0); await postTelemetry(gameEvent('round_change')); expect(testView(id)?.servers[0].progress[0].current).toBe(25); }); it('keeps Fleet reporting during pauses without crediting paused gameplay or elapsed time', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [{ ...requirement('populated-minutes', 'time'), target: 60 }] }); await tick(); await tick(); await postTelemetry(gameEvent('game_start')); vi.setSystemTime(Date.now() + 600_000); await postTelemetry(gameEvent('round_change')); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); expect(testView(id)?.servers[0].progress[0].current).toBe(10); const controls = await import('../../src/routes/api/betas/[id]/test/+server.js'); const control = (action: string) => controls.POST({ params: { id: String(id) }, request: new Request('http://test', { method: 'POST', body: JSON.stringify({ action }) }) }); expect((await control('pause')).status).toBe(200); vi.setSystemTime(Date.now() + 3_600_000); await postTelemetry(gameEvent('game_end', 50)); const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); expect(listTelemetry()[0]).toMatchObject({ event: 'game_end', round: 50 }); expect(testView(id)?.servers[0].progress[0].current).toBe(10); expect((await control('resume')).status).toBe(200); await postTelemetry(gameEvent('round_change', 1)); expect(testView(id)?.servers[0].progress[0].current).toBe(10); vi.setSystemTime(Date.now() + 600_000); await postTelemetry(gameEvent('round_change', 2)); expect(testView(id)?.servers[0].progress[0].current).toBe(20); }); it('migrates existing unsupported checks into review items and resets active progress once', async () => { const id = await openPlan({ ...defaultPlan(), requirements: [requirement('round', 'round')] }); await tick(); await tick(); await postTelemetry(gameEvent('round_change')); await tick(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const oldPlan = { ...defaultPlan(), requirements: [ requirement('round', 'round'), ...['feature', 'restart-budget', 'clean-logs'].map((kind, i) => ({ ...requirement('manual', `old-${i}`), kind, event: 'perk.purchased' })) ] }; db() .prepare('UPDATE beta_test_plans SET plan = ? WHERE beta_id = ?') .run(JSON.stringify(oldPlan), id); db() .prepare('INSERT INTO beta_test_presets (name, plan) VALUES (?, ?)') .run('Old plan', JSON.stringify(oldPlan)); db().prepare("DELETE FROM settings WHERE key = 'game_telemetry_v2'").run(); db().pragma('user_version = 0'); closeDb(); const { testView } = await import('../../src/lib/server/fleet/beta-tests.js'); const { getBeta } = await import('../../src/lib/server/fleet/beta.js'); const view = testView(id)!; expect(view.plan.requirements.map((r) => r.kind)).toEqual([ 'round', 'manual', 'manual', 'manual' ]); expect(view.plan.requirements.some((r) => 'event' in r)).toBe(false); expect(view.servers[0].progress.every((p) => !p.passed)).toBe(true); expect(view.evidence.some((e) => e.kind === 'round_change')).toBe(true); expect(getBeta(id)?.state).toBe('running'); const presets = await import('../../src/routes/api/betas/presets/+server.js'); const saved = (await presets.GET().json()).presets.find( (p: { name: string }) => p.name === 'Old plan' ); expect(saved.plan).toEqual(view.plan); await postTelemetry(gameEvent('round_change')); closeDb(); expect(testView(id)?.servers[0].progress[0].current).toBe(25); }); it('uses game_end at round 2 or later for the legacy match gate without reading logs', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta, getBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js'); const { script, version } = createScript({ releaseVersion: '1.0.0', name: 'Legacy beta', targetRelPath: 'legacy.gsc', content: Buffer.from('main() {}') }); const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [names[0]], waitForHealth: true, waitForMatch: true }); await tick(); await tick(); expect(listBetaServers(betaId)[0].stage).toBe('awaiting-match'); await appendFile(join(fixture.t5, names[0], 'logs/games_sp.log'), 'ExitLevel: executed\n'); await tick(); expect(getBeta(betaId)?.state).toBe('running'); await postTelemetry(gameEvent('game_end', 1)); await tick(); expect(listBetaServers(betaId)[0].stage).toBe('awaiting-match'); expect(getBeta(betaId)?.state).toBe('running'); await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await postTelemetry(gameEvent('game_start')); await tick(); expect(getBeta(betaId)?.state).toBe('ready-promote'); }); it('stores validated presets as copies and rejects malformed plans', async () => { const presets = await import('../../src/routes/api/betas/presets/+server.js'); const post = (plan: unknown) => presets.POST({ request: new Request('http://test', { method: 'POST', body: JSON.stringify({ name: 'My preset', plan }) }) }); expect((await post({ ...defaultPlan(), requirements: [] })).status).toBe(400); expect((await post(defaultPlan())).status).toBe(200); const response = await presets.GET().json(); expect(response.presets.find((p: { name: string }) => p.name === 'My preset').plan).toEqual( defaultPlan() ); }); });