josh
fb48f64de1
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely. Fixes #28 Validation: 805 tests passed, Svelte checks clean, production build passed.
233 lines
17 KiB
TypeScript
233 lines
17 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { existsSync } from 'node:fs';
|
|
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { dirname, join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { activateFiles } from '../helpers/staged-files.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture(servers);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); });
|
|
|
|
async function open(kind: 'script' | 'plugin' | 'baseline' = 'script') {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({ name: `Staged ${kind}`, kind,
|
|
targetRelPath: kind === 'script' ? 'nested/test.gsc' : kind === 'plugin' ? 'test.dll' : undefined,
|
|
scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'baseline' ? 'set sv_voice "1"\n' : 'beta bytes') });
|
|
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
|
|
return { betaId, script, version };
|
|
}
|
|
|
|
describe('staged beta files', () => {
|
|
it.each(['script', 'plugin', 'baseline'] as const)('queues %s bytes without creating live directories or recording deployed inventory', async kind => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const dest = destOfKind(servers[0], kind);
|
|
await rm(dest, { recursive: true, force: true });
|
|
const { betaId } = await open(kind);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(existsSync(dest)).toBe(false);
|
|
expect(db().prepare('SELECT stage, deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
|
|
.toEqual({ stage: 'awaiting-empty', deployed_at: null });
|
|
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
|
|
expect(db().prepare('SELECT * FROM beta_file_changes').all()).toHaveLength(0);
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
|
|
await activateFiles('beta', betaId);
|
|
expect(existsSync(dest)).toBe(true);
|
|
expect(db().prepare('SELECT deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
|
|
.toEqual({ deployed_at: expect.any(String) });
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(1);
|
|
});
|
|
|
|
it('cancels an unactivated beta without changing existing files or queuing a restart', async () => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'original');
|
|
const { betaId } = await open();
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(await rollbackBeta(betaId)).toMatchObject({ deploymentId: null, restored: [], dropped: [] });
|
|
expect(await readFile(path, 'utf8')).toBe('original');
|
|
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ state: 'cancelled' })]);
|
|
});
|
|
|
|
it('captures the actual activation before image and queues an unplanned beta rollback', async () => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'request-time bytes');
|
|
const { betaId } = await open();
|
|
await writeFile(path, 'activation-time bytes');
|
|
await activateFiles('beta', betaId);
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await rollbackBeta(betaId);
|
|
expect(result).toMatchObject({ deploymentId: expect.any(String), restored: [servers[0]] });
|
|
expect(await readFile(path, 'utf8')).toBe('beta bytes');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
|
|
await activateFiles('campaign', result.deploymentId!);
|
|
expect(await readFile(path, 'utf8')).toBe('activation-time bytes');
|
|
});
|
|
|
|
it('holds manual rollback files until its campaign is resumed', async () => {
|
|
const { betaId } = await open();
|
|
await activateFiles('beta', betaId);
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await rollbackBeta(betaId, { restart: 'manual' });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(result.deploymentId))
|
|
.toEqual({ restart_state: 'held' });
|
|
await expect(activateFiles('campaign', result.deploymentId!)).rejects.toThrow('no longer eligible');
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
expect(await readFile(join(destOfKind(servers[0], 'script'), 'nested/test.gsc'), 'utf8')).toBe('beta bytes');
|
|
});
|
|
|
|
it.each([false, true])('prepares missing files before retrying a beta (test plan: %s)', async planned => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
|
|
const { script, version } = createScript({ name: 'Needs baseline', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
|
|
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [servers[0]], waitForHealth: false, waitForMatch: false, ...(planned ? { testPlan: defaultPlan() } : {}) });
|
|
expect(listBetaServers(betaId)[0].stage).toBe('failed');
|
|
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
|
|
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
|
|
const retry = () => POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
|
|
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
|
|
}) });
|
|
expect((await retry()).status).toBe(planned ? 400 : 409);
|
|
expect(listBetaServers(betaId)[0].stage).toBe('failed');
|
|
const baseline = createScript({ name: 'Required baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id);
|
|
expect((await retry()).status).toBe(200);
|
|
expect(listBetaServers(betaId)[0]).toMatchObject({ stage: 'awaiting-empty', deployed_at: null });
|
|
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ kind: 'config', state: 'pending' })]);
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
|
|
});
|
|
|
|
it.each([false, true])('rejects an interrupted file write on retry (test plan: %s)', async planned => {
|
|
const { betaId } = await open();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
|
|
if (planned) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(defaultPlan()));
|
|
db().prepare("UPDATE beta_servers SET stage = 'failed' WHERE beta_id = ?").run(betaId);
|
|
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId));
|
|
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
|
|
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
|
|
const result = await POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
|
|
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
|
|
}) });
|
|
expect(result.status).toBe(planned ? 400 : 409);
|
|
expect((await result.json()).error).toContain('file write was interrupted');
|
|
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'failed' });
|
|
});
|
|
|
|
it.each([
|
|
{ operationState: null, fileState: 'pending', restartState: 'pending', blocked: true },
|
|
{ operationState: 'waiting', fileState: 'pending', restartState: 'pending', blocked: true },
|
|
{ operationState: 'failed', fileState: 'pending', restartState: 'pending', blocked: false },
|
|
{ operationState: 'cancelled', fileState: 'pending', restartState: 'pending', blocked: false },
|
|
{ operationState: 'interrupted', fileState: 'applying', restartState: 'failed', blocked: true },
|
|
{ operationState: 'cancelled', fileState: 'pending', restartState: 'held', blocked: true },
|
|
{ operationState: 'failed', fileState: 'pending', restartState: 'manual', blocked: true },
|
|
{ operationState: null, fileState: null, restartState: 'pending', blocked: false }
|
|
])('protects new beta admission from queued campaign work: %j', async ({ operationState, fileState, restartState, blocked }) => {
|
|
const { betaId, script } = await open();
|
|
await activateFiles('beta', betaId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
const { promoteBeta, openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const promotion = await promoteBeta(betaId);
|
|
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ? AND server = ?')
|
|
.run(restartState, promotion.deploymentId, servers[1]);
|
|
if (operationState) {
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const operation = requestRestart(servers[1], { kind: 'campaign', id: promotion.deploymentId });
|
|
db().prepare('UPDATE server_deployments SET state = ? WHERE id = ?').run(operationState, operation.id);
|
|
}
|
|
if (fileState) db().prepare('UPDATE staged_file_changes SET state = ? WHERE source_kind = ? AND source_id = ?')
|
|
.run(fileState, 'campaign', promotion.deploymentId);
|
|
else db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").run(promotion.deploymentId);
|
|
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const newer = uploadNewVersion(script.id, Buffer.from('newer beta bytes'));
|
|
const request = openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
|
|
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
|
|
if (blocked) await expect(request).rejects.toMatchObject({ kind: 'conflict', details: [expect.objectContaining({ deploymentId: promotion.deploymentId })] });
|
|
else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
|
|
});
|
|
|
|
it.each(['script', 'baseline'] as const)('keeps the newest %s promotion when another campaign is queued during its beta', async kind => {
|
|
const { betaId: olderBeta, script } = await open(kind);
|
|
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const newestBytes = kind === 'script' ? 'newest tested bytes' : 'set sv_voice "0"\n';
|
|
const newer = uploadNewVersion(script.id, Buffer.from(newestBytes));
|
|
const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { betaId: newerBeta } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
|
|
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
|
|
await activateFiles('beta', olderBeta);
|
|
await activateFiles('beta', newerBeta);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id IN (?, ?)").run(olderBeta, newerBeta);
|
|
const older = await promoteBeta(olderBeta);
|
|
const latest = await promoteBeta(newerBeta);
|
|
expect(latest.awaitingActivation).toBe(2);
|
|
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(stagedFileChanges('campaign', latest.deploymentId, servers[1]))
|
|
.toEqual([expect.objectContaining({ preserve_before_image: 1, state: 'pending' })]);
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId });
|
|
const second = requestRestart(servers[1], { kind: 'campaign', id: latest.deploymentId });
|
|
expect(second.id).toBe(first.id);
|
|
await activateFiles('campaign', latest.deploymentId);
|
|
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
|
const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]);
|
|
expect(await readFile(join(destOfKind(servers[1], kind), relPath), 'utf8')).toContain(newestBytes.trim());
|
|
expect(db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
|
|
.get(latest.deploymentId, servers[1])).toEqual({ content_before: null });
|
|
});
|
|
|
|
it.each([false, true])('retains the participant pre-beta image when changed composition activation is %s', async activatePromotion => {
|
|
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'config'), configFileOf(servers[0]));
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'original configuration');
|
|
const { betaId } = await open('baseline');
|
|
await activateFiles('beta', betaId);
|
|
const betaBytes = await readFile(path, 'utf8');
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { script, version } = createScript({ name: 'Later overlay', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
|
|
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await promoteBeta(betaId);
|
|
expect(result.awaitingActivation).toBe(2);
|
|
expect(await readFile(path, 'utf8')).toBe(betaBytes);
|
|
if (activatePromotion) {
|
|
await activateFiles('campaign', result.deploymentId);
|
|
expect(await readFile(path, 'utf8')).toContain('set sv_voice "0"');
|
|
}
|
|
const history = db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
|
|
.get(result.deploymentId, servers[0]) as { content_before: Buffer };
|
|
expect(history.content_before.toString()).toBe('original configuration');
|
|
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
|
const rollback = await rollbackPromoteDeployment(result.deploymentId);
|
|
expect(rollback.affectedServers).toContain(servers[0]);
|
|
expect(await readFile(path, 'utf8')).not.toBe('original configuration');
|
|
await activateFiles('campaign', rollback.deploymentId);
|
|
expect(await readFile(path, 'utf8')).toBe('original configuration');
|
|
});
|
|
});
|