Allow revising active betas with fresh tests and history
This commit is contained in:
@@ -68,6 +68,30 @@ async function tick(seconds = 15, players = 4, healthy = true) {
|
||||
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
|
||||
|
||||
describe('durable beta test plans', () => {
|
||||
it('requires fresh gameplay after a revision and promotes exactly the replacement version', async () => {
|
||||
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
|
||||
await tick(); await tick();
|
||||
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
|
||||
const { getBeta, listBetaScripts, reviseBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const { getScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const file = listBetaScripts(id)[0];
|
||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||
await reviseBeta(id, 1, [{ scriptId: file.script_id, content: Buffer.from('main() { /* fixed */ }') }]);
|
||||
const replacement = listBetaScripts(id)[0].script_version_id;
|
||||
expect(replacement).not.toBe(file.script_version_id);
|
||||
await postTelemetry(gameEvent('game_end')); // Still running the old revision.
|
||||
expect(testView(id)?.evidence).toHaveLength(0);
|
||||
await postTelemetry(gameEvent('heartbeat', 1, 0));
|
||||
await tick(); await tick();
|
||||
expect(getBeta(id)?.state).toBe('running');
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
|
||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||
await promoteBeta(id);
|
||||
expect(getScript(file.script_id)?.current_version_id).toBe(replacement);
|
||||
});
|
||||
|
||||
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
|
||||
const id = await openPlan(defaultPlan());
|
||||
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { activateFiles } from '../helpers/staged-files.js';
|
||||
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
|
||||
let fixture: Fixture;
|
||||
const names = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(names);
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
|
||||
|
||||
async function setup(options: { original?: string; waveSize?: number; kind?: 'script' | 'baseline'; legacy?: boolean } = {}) {
|
||||
const scripts = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const beta = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
||||
const kind = options.kind ?? 'script';
|
||||
const path = join(destOfKind(names[0], kind), kind === 'script' ? 'revision.gsc' : configFileOf(names[0]));
|
||||
if (options.original !== undefined) {
|
||||
await mkdir(dirname(path), { recursive: true }); await writeFile(path, options.original);
|
||||
}
|
||||
const { script, version } = scripts.createScript({ name: 'Revision test', targetRelPath: kind === 'script' ? 'revision.gsc' : undefined,
|
||||
kind, scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'script' ? 'revision one' : 'set sv_voice "1"\n') });
|
||||
const plan = { ...defaultPlan(), waveSize: options.waveSize ?? 0, requirements: [requirement('manual', 'review')] };
|
||||
const { betaId } = await beta.openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
||||
servers: options.waveSize ? names : [names[0]], waitForHealth: true, waitForMatch: false,
|
||||
...(options.legacy ? {} : { testPlan: plan }) });
|
||||
// Pause while arranging test state; revision submission explicitly starts testing.
|
||||
if (!options.legacy) db().prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
|
||||
return { ...scripts, ...beta, db: db(), script, version, betaId, path, plan };
|
||||
}
|
||||
|
||||
describe('beta revisions', () => {
|
||||
it('keeps the beta identity and archives files, sign-offs and evidence while resetting tests and the deadline', async () => {
|
||||
const x = await setup();
|
||||
await activateFiles('beta', x.betaId);
|
||||
const { prepareRun, saveMetrics, readMetrics, evidence, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const { currentBetaRevision, betaRevisionHistory } = await import('../../src/lib/server/fleet/beta-revisions.js');
|
||||
const run = await prepareRun(x.betaId, names[0]);
|
||||
const metrics = readMetrics(run); metrics.manual.review = 'Josh: verified'; saveMetrics(run, metrics);
|
||||
evidence(run, 'old-signoff', 'manual-sign-off', { note: 'Josh: verified' });
|
||||
x.db.prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(x.betaId);
|
||||
x.db.prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(x.betaId);
|
||||
const opened = x.getBeta(x.betaId)!.opened_at;
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.parse(opened) + 60_000);
|
||||
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('revision two') }], 'Small fix'))
|
||||
.resolves.toEqual({ revision: 2 });
|
||||
expect(x.getBeta(x.betaId)).toMatchObject({ state: 'running', opened_at: opened });
|
||||
expect(currentBetaRevision(x.betaId)).toEqual({ revision: 2, started_at: new Date().toISOString() });
|
||||
expect(testView(x.betaId)).toMatchObject({ outcome: 'active', servers: [], evidence: [] });
|
||||
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', restarted_at: null, deployed_at: null, provisioning_id: null });
|
||||
const history = betaRevisionHistory(x.betaId);
|
||||
expect(history[0]).toMatchObject({ revision: 2, notes: 'Small fix', results: null });
|
||||
expect(history[1].results?.test?.servers[0].passed).toBe(true);
|
||||
expect(history[1].results?.test?.evidence).toHaveLength(1);
|
||||
expect(x.db.prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = 1').get(x.betaId))
|
||||
.toEqual({ content: Buffer.from('revision one') });
|
||||
expect(await readFile(x.path, 'utf8')).toBe('revision one');
|
||||
await expect(x.promoteBeta(x.betaId)).rejects.toThrow('has not passed');
|
||||
const nextRun = x.db.prepare('SELECT revision, metrics FROM beta_test_runs WHERE retired_at IS NULL').get() as { revision: number; metrics: string };
|
||||
expect(nextRun.revision).toBe(2); expect(JSON.parse(nextRun.metrics).manual).toEqual({});
|
||||
});
|
||||
|
||||
it.each(['original bytes', undefined])('restores the pre-beta state after multiple revisions (original=%s)', async original => {
|
||||
const x = await setup({ original });
|
||||
await activateFiles('beta', x.betaId);
|
||||
for (let revision = 2; revision <= 3; revision++) {
|
||||
await x.reviseBeta(x.betaId, revision - 1, [{ scriptId: x.script.id, content: Buffer.from(`revision ${revision}`) }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
}
|
||||
expect(await readFile(x.path, 'utf8')).toBe('revision 3');
|
||||
expect(x.db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get())
|
||||
.toEqual({ had_original: original === undefined ? 0 : 1, prior_version_id: null });
|
||||
const result = await x.rollbackBeta(x.betaId);
|
||||
await activateFiles('campaign', result.deploymentId!);
|
||||
if (original === undefined) expect(existsSync(x.path)).toBe(false);
|
||||
else expect(await readFile(x.path, 'utf8')).toBe(original);
|
||||
});
|
||||
|
||||
it.each([false, true])('rolls back a prior activation while its replacement is still queued (waves=%s)', async waves => {
|
||||
const x = await setup({ original: 'before beta', waveSize: waves ? 1 : 0 });
|
||||
await activateFiles('beta', x.betaId);
|
||||
if (waves) {
|
||||
await x.deployBetaServers(x.betaId, [names[1]]); await activateFiles('beta', x.betaId);
|
||||
}
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('pending replacement') }]);
|
||||
if (waves) expect(x.listBetaServers(x.betaId).map(row => row.stage)).toEqual(['awaiting-empty', 'queued']);
|
||||
const result = await x.rollbackBeta(x.betaId);
|
||||
expect(result.restored).toContain(names[0]);
|
||||
if (waves) expect(result.dropped).toContain(names[1]);
|
||||
await activateFiles('campaign', result.deploymentId!);
|
||||
expect(await readFile(x.path, 'utf8')).toBe('before beta');
|
||||
});
|
||||
|
||||
it('does not restore or delete files when no revision was ever activated', async () => {
|
||||
const x = await setup({ original: 'untouched' });
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('new') }]);
|
||||
expect(await x.rollbackBeta(x.betaId)).toMatchObject({ deploymentId: null });
|
||||
expect(await readFile(x.path, 'utf8')).toBe('untouched');
|
||||
});
|
||||
|
||||
it('preserves the original tracked inventory version across revisions', async () => {
|
||||
const x = await setup({ original: 'tracked original' });
|
||||
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const prior = x.uploadNewVersion(x.script.id, Buffer.from('tracked original'));
|
||||
recordScript(names[0], 'revision.gsc', prior.id, prior.sha256, prior.size, 'test');
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
expect(x.db.prepare('SELECT prior_version_id FROM beta_server_scripts').get()).toEqual({ prior_version_id: prior.id });
|
||||
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
|
||||
expect(getServerScript(names[0], 'revision.gsc')?.script_version_id).toBe(prior.id);
|
||||
});
|
||||
|
||||
it('starts a revision with the same original canary selection', async () => {
|
||||
const x = await setup({ waveSize: 1 });
|
||||
await x.rollbackBeta(x.betaId);
|
||||
const { betaId } = await x.openBeta({ scripts: [{ scriptId: x.script.id, scriptVersionId: x.version.id }],
|
||||
servers: [...names].reverse(), waitForHealth: true, waitForMatch: false, testPlan: x.plan });
|
||||
x.db.prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
|
||||
await x.reviseBeta(betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
|
||||
expect(x.listBetaServers(betaId).map(row => [row.server, row.stage]))
|
||||
.toEqual([[names[0], 'queued'], [names[1], 'awaiting-empty']]);
|
||||
});
|
||||
|
||||
it('allows updates after an interrupted attempt was superseded', async () => {
|
||||
const x = await setup();
|
||||
const { requestRestart, releaseRestartRequest } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
x.db.prepare("UPDATE server_deployments SET state = 'interrupted' WHERE id = ?").run(job.id);
|
||||
releaseRestartRequest(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]))
|
||||
.resolves.toEqual({ revision: 2 });
|
||||
});
|
||||
|
||||
it('recomposes config revisions and preserves the original shared file', async () => {
|
||||
const x = await setup({ original: 'original config', kind: 'baseline' });
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('set sv_voice "0"\n') }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
expect(await readFile(x.path, 'utf8')).toContain('set sv_voice "0"');
|
||||
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
|
||||
expect(await readFile(x.path, 'utf8')).toBe('original config');
|
||||
});
|
||||
|
||||
it.each(['stale', 'identical', 'empty', 'unbundled', 'duplicate', 'closed', 'deleted', 'interrupted', 'activating'])('rejects %s updates without saving a revision', async reason => {
|
||||
const x = await setup();
|
||||
const upload = { scriptId: x.script.id, content: Buffer.from('changed') };
|
||||
if (reason === 'identical') upload.content = Buffer.from(x.version.content);
|
||||
if (reason === 'empty') upload.content = Buffer.alloc(0);
|
||||
if (reason === 'unbundled') upload.scriptId = 999;
|
||||
if (reason === 'closed') x.db.prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(x.betaId);
|
||||
if (reason === 'deleted') x.db.prepare('UPDATE beta_servers SET server_deleted_at = ? WHERE beta_id = ?').run(new Date().toISOString(), x.betaId);
|
||||
if (reason === 'interrupted') x.db.prepare("UPDATE staged_file_changes SET state = 'applying'").run();
|
||||
if (reason === 'activating') {
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
x.db.prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(job.id);
|
||||
}
|
||||
await expect(x.reviseBeta(x.betaId, reason === 'stale' ? 2 : 1, reason === 'duplicate' ? [upload, upload] : [upload])).rejects.toThrow();
|
||||
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
|
||||
expect(x.listVersions(x.script.id)).toHaveLength(1);
|
||||
expect(x.listBetaScripts(x.betaId)[0].script_version_id).toBe(x.version.id);
|
||||
});
|
||||
|
||||
it('keeps the new revision through reload, and uses its deadline rather than the beta opening time', async () => {
|
||||
const x = await setup();
|
||||
x.db.prepare('UPDATE betas SET opened_at = ?').run('2000-01-01T00:00:00Z');
|
||||
x.db.prepare("UPDATE beta_test_plans SET outcome = 'inconclusive'").run();
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
const { plannedBetaTick } = await import('../../src/lib/server/pipeline/beta-plan-runner.js');
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
await plannedBetaTick(); expect(testView(x.betaId)?.outcome).toBe('active');
|
||||
const { GET } = await import('../../src/routes/api/betas/[id]/+server.js');
|
||||
const body = await (await GET({ params: { id: String(x.betaId) } })).json();
|
||||
expect(body.revision.revision).toBe(2); expect(body.revisions).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('supports legacy betas without a test plan', async () => {
|
||||
const x = await setup({ legacy: true });
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
|
||||
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', match_seen_at: null, healthy_at: null });
|
||||
expect(x.db.prepare('SELECT * FROM beta_test_plans').all()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('rolls back the entire revision when a later upload fails validation', async () => {
|
||||
const x = await setup();
|
||||
const overlay = x.createScript({ name: 'Overlay', kind: 'config', content: Buffer.from('set sv_voice "1"\n'), scopeType: 'game', scopeValue: 't5' });
|
||||
x.db.prepare("INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, 'config')")
|
||||
.run(x.betaId, overlay.script.id, overlay.version.id);
|
||||
await expect(x.reviseBeta(x.betaId, 1, [
|
||||
{ scriptId: x.script.id, content: Buffer.from('valid new version') },
|
||||
{ scriptId: overlay.script.id, content: Buffer.from('invalid config') }
|
||||
])).rejects.toThrow('at least one');
|
||||
expect(x.listVersions(x.script.id)).toHaveLength(1);
|
||||
expect(x.listVersions(overlay.script.id)).toHaveLength(1);
|
||||
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
|
||||
expect(x.listBetaScripts(x.betaId).find(file => file.script_id === x.script.id)?.script_version_id).toBe(x.version.id);
|
||||
});
|
||||
|
||||
it('accepts reviewed multipart updates and downloads immutable revision files', async () => {
|
||||
const x = await setup();
|
||||
const { POST } = await import('../../src/routes/api/betas/[id]/revisions/+server.js');
|
||||
const form = new FormData(); form.set('expectedRevision', '1'); form.set('notes', 'Fix scoring');
|
||||
form.set(`file:${x.script.id}`, new File(['changed'], 'revision.gsc'));
|
||||
const post = () => POST({ params: { id: String(x.betaId) }, request: new Request('http://test/revisions', { method: 'POST', body: form }) });
|
||||
expect((await post()).status).toBe(200); expect((await post()).status).toBe(409);
|
||||
const { GET } = await import('../../src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.js');
|
||||
const download = await GET({ params: { id: String(x.betaId), revision: '1', scriptId: String(x.script.id) } });
|
||||
expect(await download.text()).toBe('revision one');
|
||||
});
|
||||
});
|
||||
@@ -1276,6 +1276,41 @@ describe('native provisioning cancellation and rollback', () => {
|
||||
});
|
||||
|
||||
describe('cancelled native mutation bookkeeping', () => {
|
||||
it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta, reviseBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { reportHostOperation, getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { script, version } = createScript({ name: 'Revised beta', kind: 'script', targetRelPath: 'revision.gsc',
|
||||
content: Buffer.from('revision one'), scopeType: 'game', scopeValue: 't5' });
|
||||
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
||||
servers: [job.name], waitForHealth: true, waitForMatch: false });
|
||||
async function activate(before: string | null) {
|
||||
const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
|
||||
await advance();
|
||||
const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?').get(restart.id) as { stage_operation_id: string };
|
||||
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ id: number; path: string }>;
|
||||
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
|
||||
await advance();
|
||||
// A dispatched activation must settle before revision submission.
|
||||
await expect(reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('premature update') }])).rejects.toThrow();
|
||||
const activation = (await operationFor(restart.id))!;
|
||||
reportHostOperation(hostId, activation.id, { state: 'completed', result: {
|
||||
before: files.map(file => ({ ...file, contentBase64: before === null ? null : Buffer.from(before).toString('base64') }))
|
||||
} });
|
||||
await advance();
|
||||
expect((await detail(restart.id)).state).toBe('done');
|
||||
}
|
||||
await activate(original);
|
||||
await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]);
|
||||
await activate('revision one');
|
||||
const rollback = await rollbackBeta(betaId);
|
||||
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
|
||||
.toEqual({ content: original === null ? null : Buffer.from(original) });
|
||||
});
|
||||
|
||||
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
|
||||
const job = await queue();
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
|
||||
Reference in New Issue
Block a user