josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable beta test plans and staged rollout

Browse Source
This commit is contained in:
josh
2026-09-17 21:35:28 -04:00
parent 5b5266da3a
commit a50eae9d21
26 changed files with 1518 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/beta-evaluation.test.ts
+95
View File
@@ -0,0 +1,95 @@
import { describe, expect, it } from 'vitest';
import { defaultPlan, parseBetaPlan, requirement, type BetaPlan } from '../../src/lib/beta-plan.js';
import { applySample, applySignal, coveragePassed, emptyMetrics, evaluateRequirements, type TestSignal } from '../../src/lib/server/fleet/beta-evaluation.js';
const signal = (event: TestSignal['event'], extra: Partial<TestSignal> = {}): TestSignal => ({ event, bootId: 'boot-1', sessionId: 'session-1', sequence: 0, round: 0, players: 4, name: '', ...extra });
const planWith = (...kinds: Parameters<typeof requirement>[0][]): BetaPlan => ({ ...defaultPlan(), requirements: kinds.map((k, i) => requirement(k, `r${i}`)) });
describe('beta requirement evaluation', () => {
it('validates plans without accepting unknown kinds, duplicate IDs, fractional counts or unsafe numbers', () => {
expect(parseBetaPlan(defaultPlan())).toEqual(defaultPlan());
expect(() => parseBetaPlan({ ...defaultPlan(), requirements: [] })).toThrow();
expect(() => parseBetaPlan({ ...defaultPlan(), timeoutHours: NaN })).toThrow();
expect(() => parseBetaPlan({ ...defaultPlan(), requirements: [requirement('round', 'same'), requirement('round', 'same')] })).toThrow();
expect(() => parseBetaPlan({ ...defaultPlan(), requirements: [{ ...requirement('round', 'r'), target: 1.5 }] })).toThrow();
expect(() => parseBetaPlan({ ...defaultPlan(), requirements: [requirement('manual', 'constructor')] })).toThrow();
});
it('counts only qualifying rounds and completed matches from the same session', () => {
const plan = planWith('round', 'matches');
plan.requirements[0].minPlayers = 4;
plan.requirements[1] = { ...plan.requirements[1], target: 2, minRound: 25, minPlayers: 4, minMinutes: 1 };
const m = emptyMetrics();
expect(applySignal(plan, m, signal('round_change', { sequence: 1, round: 50 }), 0)).toBe(false);
applySignal(plan, m, signal('boot'), 0);
applySignal(plan, m, signal('session_start'), 0);
applySignal(plan, m, signal('round_change', { sequence: 1, round: 30, players: 1 }), 1000);
applySignal(plan, m, signal('match_end', { sequence: 2 }), 61_000);
expect(m.values.r0 ?? 0).toBe(0);
expect(m.values.r1 ?? 0).toBe(0);
applySignal(plan, m, signal('session_start', { sessionId: 'session-2' }), 62_000);
applySignal(plan, m, signal('round_change', { sessionId: 'session-2', sequence: 1, round: 25 }), 63_000);
applySignal(plan, m, signal('match_end', { sessionId: 'session-2', sequence: 2 }), 123_000);
expect(m.values).toMatchObject({ r0: 25, r1: 1 });
expect(applySignal(plan, m, signal('match_end', { sessionId: 'session-2', sequence: 3 }), 124_000)).toBe(false);
});
it('rejects out-of-order events and retired boots, while counting actual boot changes once', () => {
const plan = planWith('feature', 'restart-budget');
plan.requirements[0].event = 'perk.purchased';
const m = emptyMetrics();
applySignal(plan, m, signal('boot'), 0);
applySignal(plan, m, signal('session_start'), 0);
applySignal(plan, m, signal('feature', { sequence: 2, name: 'perk.purchased' }), 1);
expect(applySignal(plan, m, signal('feature', { sequence: 1, name: 'perk.purchased' }), 2)).toBe(false);
applySignal(plan, m, signal('boot', { bootId: 'boot-2' }), 3);
expect(applySignal(plan, m, signal('boot'), 4)).toBe(false);
expect(m.values.r0).toBe(1);
expect(evaluateRequirements(plan, m, 0)[1].failed).toBe(true);
expect(m.restarts).toBe(1);
});
it('does not double count unqualified matches across log and telemetry sources', () => {
const plan = planWith('matches');
const m = emptyMetrics();
applySignal(plan, m, signal('boot'), 0);
applySignal(plan, m, signal('session_start'), 0);
applySignal(plan, m, signal('match_end', { sequence: 1 }), 1000);
expect(m.values.r0).toBeUndefined(); // these requirements use incremental match logs
});
it('integrates populated time and player-hours conservatively and excludes monitoring gaps', () => {
const plan = planWith('populated-minutes', 'player-hours', 'healthy-minutes', 'population', 'sustained-population', 'failure-budget');
plan.requirements[4].minPlayers = 4;
const m = emptyMetrics();
applySample(plan, m, 0, true, 4);
applySample(plan, m, 30_000, true, 4);
applySample(plan, m, 60_000, true, 4);
expect(m.values).toMatchObject({ r0: 1, r1: 4 / 60, r2: 1, r3: 4, r4: 1 });
applySample(plan, m, 300_000, true, 4);
expect(m.values.r0).toBe(1);
expect(m.values.r2).toBe(0);
applySample(plan, m, 330_000, false, 0);
applySample(plan, m, 360_000, false, 0);
expect(m.outages).toBe(1);
expect(evaluateRequirements(plan, m, 0)[5].failed).toBe(true);
});
it('requires readable logs, boot evidence and explicit manual sign-off', () => {
const plan = planWith('clean-logs', 'restart-budget', 'manual');
const m = emptyMetrics();
expect(evaluateRequirements(plan, m, 0).every(r => !r.passed)).toBe(true);
m.logAvailable = true; m.bootId = 'boot'; m.manual.r2 = 'Josh: HUD checked';
expect(evaluateRequirements(plan, m, 0).every(r => r.passed)).toBe(true);
m.logErrors = 1;
expect(evaluateRequirements(plan, m, 0)[0].failed).toBe(true);
});
it('requires each selected coverage group even when many servers on one map pass', () => {
const servers = [{ game: 't5', map: 'a', passed: true }, { game: 't5', map: 'a', passed: true }, { game: 't6', map: 'b', passed: false }];
expect(coveragePassed({ ...defaultPlan(), scope: 'quorum', quorum: 2 }, servers)).toBe(true);
for (const scope of ['all', 'map', 'game', 'map-game'] as const) expect(coveragePassed({ ...defaultPlan(), scope }, servers)).toBe(false);
servers[2].passed = true;
expect(coveragePassed({ ...defaultPlan(), scope: 'map-game' }, servers)).toBe(true);
});
});
tests/unit/beta-plans.test.ts
+234
View File
@@ -0,0 +1,234 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { defaultPlan, requirement, type BetaPlan } from '../../src/lib/beta-plan.js';
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
let fixture: Fixture;
const names = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
const now = () => new Date().toISOString();
beforeEach(async () => {
vi.resetModules(); vi.resetAllMocks(); vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
fixture = await newFixture(names);
for (const name of names) {
await mkdir(join(fixture.t5, name, 'logs'), { recursive: true });
await writeFile(join(fixture.t5, name, 'logs/console.log'), 'old script error\n');
await writeFile(join(fixture.t5, name, 'logs/games_sp.log'), 'InitGame:\n 0:00 J;old;1;old\nExitLevel: executed\n');
}
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test');
vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 1 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.useRealTimers(); });
async function openPlan(plan: BetaPlan, servers = [names[0]]) {
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { script } = createScript({ name: 'Beta test', targetRelPath: 'test.gsc', content: Buffer.from('main() {}') });
const result = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }], servers, waitForHealth: true, waitForMatch: false, testPlan: plan });
db().prepare('UPDATE betas SET opened_at = ? WHERE id = ?').run(now(), result.betaId);
return result.betaId;
}
async function tick(seconds = 15, players = 4, healthy = true) {
vi.setSystemTime(Date.now() + seconds * 1000);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET last_checked_at = ?, last_seen_at = ?, last_players = ?, last_error = ?, container_state = 'running'")
.run(now(), now(), players, healthy ? null : 'timeout');
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
await betaTick();
}
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
describe('durable beta test plans', () => {
it('holds file deployment until each wave passes, and rollback leaves undeployed files alone', async () => {
await writeFile(scriptPath(names[1]), 'original untracked file');
const id = await openPlan({ ...defaultPlan(), waveSize: 1 }, names);
expect(await readFile(scriptPath(names[1]), 'utf8')).toBe('original untracked file');
const { listBetaServers, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
expect(listBetaServers(id).map(s => s.stage)).toEqual(['awaiting-empty', 'queued']);
await rollbackBeta(id);
expect(await readFile(scriptPath(names[1]), 'utf8')).toBe('original untracked file');
expect(existsSync(scriptPath(names[0]))).toBe(false);
});
it('expands only after observed healthy time, then requires the next wave to pass', async () => {
const id = await openPlan({ ...defaultPlan(), waveSize: 1 }, names);
const { getBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
await tick(); await tick();
expect(existsSync(scriptPath(names[1]))).toBe(false);
for (let i = 0; i < 4; i++) await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
expect(getBeta(id)?.state).toBe('running');
expect(listBetaServers(id)[1].stage).toBe('awaiting-empty');
for (let i = 0; i < 6; i++) await tick();
expect(getBeta(id)?.state).toBe('ready-promote');
});
it('ignores pre-beta log markers, handles partial lines, and persists cursors across app restarts', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches'), requirement('clean-logs', 'logs')] });
await tick(); await tick();
const { getBeta } = await import('../../src/lib/server/fleet/beta.js');
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
expect(getBeta(id)?.state).toBe('running');
expect(testView(id)?.servers[0].progress.map(p => p.current)).toEqual([0, 0]);
const log = join(fixture.t5, names[0], 'logs/games_sp.log');
await appendFile(log, 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: exe');
await tick();
expect(getBeta(id)?.state).toBe('running');
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
await appendFile(log, 'cuted\n');
await tick(); await tick();
expect(getBeta(id)?.state).toBe('ready-promote');
expect(testView(id)?.servers[0].progress[0].current).toBe(1);
await appendFile(join(fixture.t5, names[0], 'logs/console.log'), 'script runtime error: undefined variable\n');
await tick();
expect(getBeta(id)?.state).toBe('running');
expect(testView(id)?.outcome).toBe('failed');
});
it('accepts durable run-scoped round evidence and rejects stale runs and duplicate events', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('round', 'round')] });
await tick(); await tick();
const { acceptTestSignal, currentRun, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
const run = currentRun(id, names[0])!;
const base = { bootId: 'boot', sessionId: 'session', sequence: 0, round: 0, players: 4, name: '' };
expect(acceptTestSignal(names[0], 'stale-run', 'old', { ...base, event: 'boot' }).accepted).toBe(false);
expect(acceptTestSignal(names[0], run.id, 'boot', { ...base, event: 'boot' }).accepted).toBe(true);
expect(acceptTestSignal(names[0], run.id, 'start', { ...base, event: 'session_start' }).accepted).toBe(true);
const round = { ...base, event: 'round_change' as const, sequence: 1, round: 25 };
expect(acceptTestSignal(names[0], run.id, 'round', round).accepted).toBe(true);
expect(acceptTestSignal(names[0], run.id, 'round', round).accepted).toBe(false);
await tick();
expect(testView(id)?.outcome).toBe('passed');
expect(testView(id)?.evidence.filter(e => e.kind === 'round_change')).toHaveLength(1);
});
it('resets evidence after planned recovery and only passes after testing the new process', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('population', 'players'), requirement('recovery', 'recovery')] });
await tick(); await tick();
const { currentRun, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
const first = currentRun(id, names[0])!;
expect(testView(id)?.outcome).toBe('active');
await tick();
expect(currentRun(id, names[0])?.id).not.toBe(first.id);
expect(currentRun(id, names[0])?.phase).toBe(1);
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
await tick();
expect(testView(id)?.outcome).toBe('passed');
});
it('treats missing evidence as inconclusive and refuses promotion', async () => {
const id = await openPlan({ ...defaultPlan(), timeoutHours: 0.01, requirements: [requirement('round', 'round')] });
await tick(); await tick(); await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
expect(testView(id)?.outcome).toBe('inconclusive');
await expect(promoteBeta(id)).rejects.toThrow('Test plan has not passed');
});
it('revokes readiness if health fails while awaiting manual promotion', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('population', 'players'), requirement('failure-budget', 'outages')] });
await tick(); await tick();
const { getBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
expect(getBeta(id)?.state).toBe('ready-promote');
await tick(15, 0, false);
expect(getBeta(id)?.state).toBe('running');
await expect(promoteBeta(id)).rejects.toThrow();
});
it('automatically rolls back only the deployed wave and queues safe rollback restarts', async () => {
await writeFile(scriptPath(names[0]), 'original');
const id = await openPlan({ ...defaultPlan(), waveSize: 1, failureAction: 'rollback', requirements: [requirement('round', 'round'), requirement('clean-logs', 'logs')] }, names);
await tick(); await tick();
await appendFile(join(fixture.t5, names[0], 'logs/console.log'), 'fatal error: test\n');
await tick();
const { getBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(getBeta(id)?.state).toBe('rolled-back');
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
expect(existsSync(scriptPath(names[1]))).toBe(false);
expect(db().prepare("SELECT server, restart_state FROM deployment_servers").all()).toEqual([{ server: names[0], restart_state: 'pending' }]);
});
it('keeps snapshots and holds after a paused failure, without deploying subsequent waves', async () => {
const id = await openPlan({ ...defaultPlan(), waveSize: 1, failureAction: 'pause', requirements: [requirement('round', 'round'), requirement('failure-budget', 'outages')] }, names);
await tick(); await tick(); await tick(15, 0, false); await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
expect(testView(id)?.outcome).toBe('paused');
expect(existsSync(scriptPath(names[0]))).toBe(true);
expect(existsSync(scriptPath(names[1]))).toBe(false);
});
it('retains artifact ownership while ready for promotion', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('population', 'players')] });
await tick(); await tick();
const { heldByBeta, ownerOf } = await import('../../src/lib/server/fleet/servers.js');
expect(heldByBeta(names[0]).get('test.gsc')).toBe(id);
expect(ownerOf(names[0], 'test.gsc')).toBe(id);
const { promoteBeta, getBeta } = await import('../../src/lib/server/fleet/beta.js');
await promoteBeta(id);
expect(getBeta(id)?.state).toBe('promoted');
expect(ownerOf(names[0], 'test.gsc')).toBeNull();
});
it('does not carry joins into a truncated log or deduplicate a genuinely new match after rotation', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [{ ...requirement('matches', 'matches'), target: 2 }] });
await tick(); await tick();
const log = join(fixture.t5, names[0], 'logs/games_sp.log');
await appendFile(log, 'InitGame:\n 0:00 J;first;1;Josh\nExitLevel: executed\n');
await tick();
// Same inode, contents replaced with a larger file: anchor detects truncation/regrowth.
await writeFile(log, 'InitGame:\n 0:00 J;second;1;Josh\n' + 'padding\n'.repeat(30) + 'ExitLevel: executed\n');
await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
expect(testView(id)?.servers[0].progress[0].current).toBe(2);
expect(testView(id)?.outcome).toBe('passed');
});
it('exposes current run tokens, validates session payloads, and records checklist sign-off through the API', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('round', 'round'), requirement('manual', 'hud')] });
await tick(); await tick();
const telemetry = await import('../../src/routes/api/telemetry/+server.js');
const context = await (await telemetry.GET({ url: new URL(`http://test/api/telemetry?server=${names[0]}`) })).json();
expect(context.runs).toHaveLength(1);
const body = { server: names[0], runId: context.runs[0].runId, eventId: 'boot', bootId: 'boot', sequence: 0, round: 0, players: 0, event: 'boot' };
const post = (payload: unknown) => telemetry.POST({ request: new Request('http://test/api/telemetry', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }) });
expect((await post({ ...body, eventId: null })).status).toBe(400);
expect(await (await post(body)).json()).toMatchObject({ test: { accepted: true } });
await post({ ...body, eventId: 'start', event: 'session_start', sessionId: 'game', players: 4 });
await post({ ...body, eventId: 'round', event: 'round_change', sessionId: 'game', sequence: 1, round: 25, players: 4 });
const controls = await import('../../src/routes/api/betas/[id]/test/+server.js');
const result = await controls.POST({ params: { id: String(id) }, request: new Request('http://test', { method: 'POST', body: JSON.stringify({ action: 'sign-off', server: names[0], requirementId: 'hud', note: 'Josh: HUD verified' }) }) });
expect(result.status).toBe(200);
await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
expect(testView(id)?.outcome).toBe('passed');
const retry = await controls.POST({ params: { id: String(id) }, request: new Request('http://test', { method: 'POST', body: JSON.stringify({ action: 'retry' }) }) });
expect(retry.status).toBe(200);
expect(await (await post({ ...body, eventId: 'late' })).json()).toMatchObject({ test: { accepted: false } });
expect(testView(id)?.servers).toHaveLength(0);
});
it('stores validated presets as copies and rejects malformed plans', async () => {
const presets = await import('../../src/routes/api/betas/presets/+server.js');
const post = (plan: unknown) => presets.POST({ request: new Request('http://test', { method: 'POST', body: JSON.stringify({ name: 'My preset', plan }) }) });
expect((await post({ ...defaultPlan(), requirements: [] })).status).toBe(400);
expect((await post(defaultPlan())).status).toBe(200);
const response = await presets.GET().json();
expect(response.presets.find((p: { name: string }) => p.name === 'My preset').plan).toEqual(defaultPlan());
});
});