josh
fb48f64de1
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely. Fixes #28 Validation: 805 tests passed, Svelte checks clean, production build passed.
134 lines
8.6 KiB
TypeScript
134 lines
8.6 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { activateFiles } from '../helpers/staged-files.js';
|
|
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
|
|
import { gameEvent } from '../helpers/telemetry.js';
|
|
|
|
let fixture: Fixture;
|
|
const names = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
|
|
const now = () => new Date().toISOString();
|
|
const path = (server: string) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/history.gsc');
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture(names);
|
|
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', '');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
|
|
|
async function open(planned = false, servers = [names[0]]) {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({ name: 'Retained history', targetRelPath: 'history.gsc',
|
|
scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
|
|
const input = { scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers,
|
|
waitForHealth: false, waitForMatch: false,
|
|
...(planned ? { testPlan: { ...defaultPlan(), requirements: [requirement('round', 'round')] } } : {}) };
|
|
const { betaId } = await openBeta(input);
|
|
return { betaId, input };
|
|
}
|
|
|
|
async function replace(server: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
|
await seedServerRegistry();
|
|
await writeFile(path(server), 'replacement content');
|
|
}
|
|
|
|
describe('deleted server operational guards', () => {
|
|
it('does not credit replacement matches to a legacy beta or keep its old server hold', async () => {
|
|
const { betaId, input } = await open();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
const { openBeta, listBetaServers, getBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
db().prepare("UPDATE beta_servers SET stage = 'awaiting-match', healthy_at = ? WHERE beta_id = ?").run(now(), betaId);
|
|
await replace(names[0]);
|
|
saveTelemetry({ ...gameEvent('game_end', 99), receivedAt: now() });
|
|
await betaTick();
|
|
expect(listBetaServers(betaId)[0]).toMatchObject({ stage: 'awaiting-match', match_seen_at: null, server_deleted_at: expect.any(String) });
|
|
expect(getBeta(betaId)?.state).toBe('running');
|
|
await expect(openBeta(input)).resolves.toMatchObject({ servers: [names[0]] });
|
|
});
|
|
|
|
it('freezes test evidence, pauses original coverage, and releases the replacement for new beta work', async () => {
|
|
const { betaId, input } = await open(true);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { prepareRun, recordGameEvent, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { deployBetaServers, rollbackBeta, openBeta, listBetaServers, getBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { heldByBeta } = await import('../../src/lib/server/fleet/servers.js');
|
|
await prepareRun(betaId, names[0]);
|
|
db().prepare("UPDATE beta_servers SET stage = 'observing', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
|
|
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ?').run(now(), betaId);
|
|
recordGameEvent({ ...gameEvent('round_change', 5), receivedAt: now() });
|
|
const before = testView(betaId)!;
|
|
expect(before.servers[0].progress[0].current).toBe(5);
|
|
|
|
await replace(names[0]);
|
|
recordGameEvent({ ...gameEvent('round_change', 99), receivedAt: now() });
|
|
await betaTick();
|
|
const after = testView(betaId)!;
|
|
expect(after).toMatchObject({ outcome: 'paused', reason: expect.stringContaining('Test server deleted') });
|
|
expect(after.servers[0]).toMatchObject({ server_deleted_at: expect.any(String), progress: before.servers[0].progress });
|
|
expect(after.evidence).toHaveLength(before.evidence.length);
|
|
expect(after.evidence[0].server_deleted_at).toEqual(expect.any(String));
|
|
expect(getBeta(betaId)?.state).toBe('running');
|
|
expect(listBetaServers(betaId)[0].stage).toBe('observing');
|
|
expect(heldByBeta(names[0]).size).toBe(0);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
await expect(deployBetaServers(betaId, [names[0]])).rejects.toThrow('original beta server was deleted');
|
|
await rollbackBeta(betaId);
|
|
expect(await readFile(path(names[0]), 'utf8')).toBe('replacement content');
|
|
await expect(openBeta(input)).resolves.toMatchObject({ servers: [names[0]] });
|
|
});
|
|
|
|
it('does not treat a same-name replacement as tested or promote from deleted coverage', async () => {
|
|
const { betaId } = await open();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
db().prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(betaId);
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
const reviewed = previewBetaPromotion(betaId);
|
|
await replace(names[0]);
|
|
const preview = previewBetaPromotion(betaId);
|
|
expect(preview.servers.find(server => server.name === names[0])?.tested).toBe(false);
|
|
expect(preview.blockers).toEqual([expect.stringContaining('original test server was deleted')]);
|
|
await expect(promoteBeta(betaId, reviewed.token)).rejects.toMatchObject({ kind: 'preview-changed' });
|
|
await expect(promoteBeta(betaId, preview.token)).rejects.toMatchObject({ kind: 'invalid-state' });
|
|
expect(await readFile(path(names[0]), 'utf8')).toBe('replacement content');
|
|
expect(db().prepare('SELECT id FROM deployments').all()).toEqual([]);
|
|
});
|
|
|
|
it('skips deleted campaign targets during restart and rollback while restoring live targets', async () => {
|
|
for (const name of names) await writeFile(path(name), 'original content');
|
|
const { betaId } = await open();
|
|
await activateFiles('beta', betaId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
const campaign = await promoteBeta(betaId);
|
|
const provisioningBefore = db().prepare('SELECT id FROM server_deployments').all();
|
|
await replace(names[1]);
|
|
await sweep();
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual(provisioningBefore);
|
|
expect(() => requestRestart(names[1], { kind: 'campaign', id: campaign.deploymentId })).toThrow('Restart source no longer exists');
|
|
const result = await rollbackPromoteDeployment(campaign.deploymentId);
|
|
expect(result.affectedServers).toEqual([names[0]]);
|
|
expect(await readFile(path(names[0]), 'utf8')).toBe('beta content');
|
|
await activateFiles('campaign', result.deploymentId);
|
|
expect(await readFile(path(names[0]), 'utf8')).toBe('original content');
|
|
expect(await readFile(path(names[1]), 'utf8')).toBe('replacement content');
|
|
expect(db().prepare('SELECT server FROM deployment_servers WHERE deployment_id = ?').all(result.deploymentId)).toEqual([{ server: names[0] }]);
|
|
});
|
|
});
|