408 lines
20 KiB
TypeScript
408 lines
20 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { existsSync } from 'node:fs';
|
|
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { dirname, join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { activateFiles } from '../helpers/staged-files.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture(servers);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
async function open(kind: 'script' | 'plugin' | 'baseline' = 'script') {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { script, version } = createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: `Staged ${kind}`,
|
|
kind,
|
|
targetRelPath:
|
|
kind === 'script' ? 'nested/test.gsc' : kind === 'plugin' ? 'test.dll' : undefined,
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from(kind === 'baseline' ? 'set sv_voice "1"\n' : 'beta bytes')
|
|
});
|
|
const { betaId } = await openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [servers[0]],
|
|
waitForHealth: false,
|
|
waitForMatch: false
|
|
});
|
|
return { betaId, script, version };
|
|
}
|
|
|
|
describe('staged beta files', () => {
|
|
it.each(['script', 'plugin', 'baseline'] as const)(
|
|
'queues %s bytes without creating live directories or recording deployed inventory',
|
|
async (kind) => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const dest = destOfKind(servers[0], kind);
|
|
await rm(dest, { recursive: true, force: true });
|
|
const { betaId } = await open(kind);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(existsSync(dest)).toBe(false);
|
|
expect(
|
|
db()
|
|
.prepare('SELECT stage, deployed_at FROM beta_servers WHERE beta_id = ?')
|
|
.get(betaId)
|
|
).toEqual({ stage: 'awaiting-empty', deployed_at: null });
|
|
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
|
|
expect(db().prepare('SELECT * FROM beta_file_changes').all()).toHaveLength(0);
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
|
|
await activateFiles('beta', betaId);
|
|
expect(existsSync(dest)).toBe(true);
|
|
expect(
|
|
db().prepare('SELECT deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId)
|
|
).toEqual({ deployed_at: expect.any(String) });
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(1);
|
|
}
|
|
);
|
|
|
|
it('cancels an unactivated beta without changing existing files or queuing a restart', async () => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'original');
|
|
const { betaId } = await open();
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(await rollbackBeta(betaId)).toMatchObject({
|
|
deploymentId: null,
|
|
restored: [],
|
|
dropped: []
|
|
});
|
|
expect(await readFile(path, 'utf8')).toBe('original');
|
|
expect(stagedFileChanges('beta', String(betaId))).toEqual([
|
|
expect.objectContaining({ state: 'cancelled' })
|
|
]);
|
|
});
|
|
|
|
it('captures the actual activation before image and queues an unplanned beta rollback', async () => {
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'request-time bytes');
|
|
const { betaId } = await open();
|
|
await writeFile(path, 'activation-time bytes');
|
|
await activateFiles('beta', betaId);
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await rollbackBeta(betaId);
|
|
expect(result).toMatchObject({ deploymentId: expect.any(String), restored: [servers[0]] });
|
|
expect(await readFile(path, 'utf8')).toBe('beta bytes');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
|
|
await activateFiles('campaign', result.deploymentId!);
|
|
expect(await readFile(path, 'utf8')).toBe('activation-time bytes');
|
|
});
|
|
|
|
it('holds manual rollback files until its campaign is resumed', async () => {
|
|
const { betaId } = await open();
|
|
await activateFiles('beta', betaId);
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await rollbackBeta(betaId, { restart: 'manual' });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?')
|
|
.get(result.deploymentId)
|
|
).toEqual({ restart_state: 'held' });
|
|
await expect(activateFiles('campaign', result.deploymentId!)).rejects.toThrow(
|
|
'no longer eligible'
|
|
);
|
|
const { destOfKind } = await import('../../src/lib/server/config.js');
|
|
expect(
|
|
await readFile(join(destOfKind(servers[0], 'script'), 'nested/test.gsc'), 'utf8')
|
|
).toBe('beta bytes');
|
|
});
|
|
|
|
it.each([false, true])(
|
|
'prepares missing files before retrying a beta (test plan: %s)',
|
|
async (planned) => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta, listBetaServers } =
|
|
await import('../../src/lib/server/fleet/beta.js');
|
|
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
|
|
const { script, version } = createScript({
|
|
name: 'Needs baseline',
|
|
kind: 'config',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('set sv_voice "0"\n')
|
|
});
|
|
const { betaId } = await openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
|
servers: [servers[0]],
|
|
waitForHealth: false,
|
|
waitForMatch: false,
|
|
...(planned ? { testPlan: defaultPlan() } : {})
|
|
});
|
|
expect(listBetaServers(betaId)[0].stage).toBe('failed');
|
|
const { POST } = planned
|
|
? await import('../../src/routes/api/betas/[id]/test/+server.js')
|
|
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
|
|
const retry = () =>
|
|
POST({
|
|
params: { id: String(betaId) },
|
|
request: new Request('http://test/retry', {
|
|
method: 'POST',
|
|
body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
|
|
})
|
|
});
|
|
expect((await retry()).status).toBe(planned ? 400 : 409);
|
|
expect(listBetaServers(betaId)[0].stage).toBe('failed');
|
|
const baseline = createScript({
|
|
name: 'Required baseline',
|
|
kind: 'baseline',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('set sv_voice "1"\n')
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?')
|
|
.run(baseline.version.id, baseline.script.id);
|
|
expect((await retry()).status).toBe(200);
|
|
expect(listBetaServers(betaId)[0]).toMatchObject({
|
|
stage: 'awaiting-empty',
|
|
deployed_at: null
|
|
});
|
|
const { stagedFileChanges } =
|
|
await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(stagedFileChanges('beta', String(betaId))).toEqual([
|
|
expect.objectContaining({ kind: 'config', state: 'pending' })
|
|
]);
|
|
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
|
|
}
|
|
);
|
|
|
|
it.each([false, true])(
|
|
'rejects an interrupted file write on retry (test plan: %s)',
|
|
async (planned) => {
|
|
const { betaId } = await open();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
|
|
if (planned)
|
|
db()
|
|
.prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)')
|
|
.run(betaId, JSON.stringify(defaultPlan()));
|
|
db().prepare("UPDATE beta_servers SET stage = 'failed' WHERE beta_id = ?").run(betaId);
|
|
db()
|
|
.prepare(
|
|
"UPDATE staged_file_changes SET state = 'applying' WHERE source_kind = 'beta' AND source_id = ?"
|
|
)
|
|
.run(String(betaId));
|
|
const { POST } = planned
|
|
? await import('../../src/routes/api/betas/[id]/test/+server.js')
|
|
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
|
|
const result = await POST({
|
|
params: { id: String(betaId) },
|
|
request: new Request('http://test/retry', {
|
|
method: 'POST',
|
|
body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
|
|
})
|
|
});
|
|
expect(result.status).toBe(planned ? 400 : 409);
|
|
expect((await result.json()).error).toContain('file write was interrupted');
|
|
expect(
|
|
db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)
|
|
).toEqual({ stage: 'failed' });
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
{ operationState: null, fileState: 'pending', restartState: 'pending', blocked: true },
|
|
{ operationState: 'waiting', fileState: 'pending', restartState: 'pending', blocked: true },
|
|
{ operationState: 'failed', fileState: 'pending', restartState: 'pending', blocked: false },
|
|
{
|
|
operationState: 'cancelled',
|
|
fileState: 'pending',
|
|
restartState: 'pending',
|
|
blocked: false
|
|
},
|
|
{
|
|
operationState: 'interrupted',
|
|
fileState: 'applying',
|
|
restartState: 'failed',
|
|
blocked: true
|
|
},
|
|
{ operationState: 'cancelled', fileState: 'pending', restartState: 'held', blocked: true },
|
|
{ operationState: 'failed', fileState: 'pending', restartState: 'manual', blocked: true },
|
|
{ operationState: null, fileState: null, restartState: 'pending', blocked: false }
|
|
])(
|
|
'protects new beta admission from queued campaign work: %j',
|
|
async ({ operationState, fileState, restartState, blocked }) => {
|
|
const { betaId, script } = await open();
|
|
await activateFiles('beta', betaId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
const { promoteBeta, openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const promotion = await promoteBeta(betaId);
|
|
db()
|
|
.prepare(
|
|
'UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ? AND server = ?'
|
|
)
|
|
.run(restartState, promotion.deploymentId, servers[1]);
|
|
if (operationState) {
|
|
const { requestRestart } =
|
|
await import('../../src/lib/server/provisioning/index.js');
|
|
const operation = requestRestart(servers[1], {
|
|
kind: 'campaign',
|
|
id: promotion.deploymentId
|
|
});
|
|
db()
|
|
.prepare('UPDATE server_deployments SET state = ? WHERE id = ?')
|
|
.run(operationState, operation.id);
|
|
}
|
|
if (fileState)
|
|
db()
|
|
.prepare(
|
|
'UPDATE staged_file_changes SET state = ? WHERE source_kind = ? AND source_id = ?'
|
|
)
|
|
.run(fileState, 'campaign', promotion.deploymentId);
|
|
else
|
|
db()
|
|
.prepare(
|
|
"DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?"
|
|
)
|
|
.run(promotion.deploymentId);
|
|
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const newer = uploadNewVersion(
|
|
script.id,
|
|
Buffer.from('newer beta bytes'),
|
|
undefined,
|
|
'1.0.1'
|
|
);
|
|
const request = openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
|
|
servers: [servers[1]],
|
|
waitForHealth: false,
|
|
waitForMatch: false
|
|
});
|
|
if (blocked)
|
|
await expect(request).rejects.toMatchObject({
|
|
kind: 'conflict',
|
|
...(operationState === 'waiting'
|
|
? { message: expect.stringContaining('rebuild') }
|
|
: {
|
|
details: [
|
|
expect.objectContaining({ deploymentId: promotion.deploymentId })
|
|
]
|
|
})
|
|
});
|
|
else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
|
|
}
|
|
);
|
|
|
|
it.each(['script', 'baseline'] as const)(
|
|
'keeps the newest %s promotion when another campaign is queued during its beta',
|
|
async (kind) => {
|
|
const { betaId: olderBeta, script } = await open(kind);
|
|
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const newestBytes = kind === 'script' ? 'newest tested bytes' : 'set sv_voice "0"\n';
|
|
const newer = uploadNewVersion(script.id, Buffer.from(newestBytes), undefined, '1.0.2');
|
|
const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { betaId: newerBeta } = await openBeta({
|
|
scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
|
|
servers: [servers[1]],
|
|
waitForHealth: false,
|
|
waitForMatch: false
|
|
});
|
|
await activateFiles('beta', olderBeta);
|
|
await activateFiles('beta', newerBeta);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare("UPDATE betas SET state = 'ready-promote' WHERE id IN (?, ?)")
|
|
.run(olderBeta, newerBeta);
|
|
const older = await promoteBeta(olderBeta);
|
|
const latest = await promoteBeta(newerBeta);
|
|
expect(latest.awaitingActivation).toBe(2);
|
|
const { stagedFileChanges } =
|
|
await import('../../src/lib/server/fleet/staged-files.js');
|
|
expect(stagedFileChanges('campaign', latest.deploymentId, servers[1])).toEqual([
|
|
expect.objectContaining({ preserve_before_image: 1, state: 'pending' })
|
|
]);
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId });
|
|
const second = requestRestart(servers[1], {
|
|
kind: 'campaign',
|
|
id: latest.deploymentId
|
|
});
|
|
expect(second.id).not.toBe(first.id);
|
|
await activateFiles('campaign', latest.deploymentId);
|
|
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
|
const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]);
|
|
expect(await readFile(join(destOfKind(servers[1], kind), relPath), 'utf8')).toContain(
|
|
newestBytes.trim()
|
|
);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files WHERE deployment_id = ? AND server = ?'
|
|
)
|
|
.get(latest.deploymentId, servers[1])
|
|
).toEqual({ content_before: null });
|
|
}
|
|
);
|
|
|
|
it.each([false, true])(
|
|
'retains the participant pre-beta image when changed composition activation is %s',
|
|
async (activatePromotion) => {
|
|
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
|
const path = join(destOfKind(servers[0], 'config'), configFileOf(servers[0]));
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await writeFile(path, 'original configuration');
|
|
const { betaId } = await open('baseline');
|
|
await activateFiles('beta', betaId);
|
|
const betaBytes = await readFile(path, 'utf8');
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { script, version } = createScript({
|
|
name: 'Later overlay',
|
|
kind: 'config',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('set sv_voice "0"\n')
|
|
});
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?')
|
|
.run(version.id, script.id);
|
|
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
|
db()
|
|
.prepare(
|
|
"UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?"
|
|
)
|
|
.run(new Date().toISOString(), betaId);
|
|
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await promoteBeta(betaId);
|
|
expect(result.awaitingActivation).toBe(2);
|
|
expect(await readFile(path, 'utf8')).toBe(betaBytes);
|
|
if (activatePromotion) {
|
|
await activateFiles('campaign', result.deploymentId);
|
|
expect(await readFile(path, 'utf8')).toContain('set sv_voice "0"');
|
|
}
|
|
const history = db()
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before FROM deployment_files WHERE deployment_id = ? AND server = ?'
|
|
)
|
|
.get(result.deploymentId, servers[0]) as { content_before: Buffer };
|
|
expect(history.content_before.toString()).toBe('original configuration');
|
|
const { rollbackPromoteDeployment } =
|
|
await import('../../src/lib/server/fleet/rollback.js');
|
|
const rollback = await rollbackPromoteDeployment(result.deploymentId);
|
|
expect(rollback.affectedServers).toContain(servers[0]);
|
|
expect(await readFile(path, 'utf8')).not.toBe('original configuration');
|
|
await activateFiles('campaign', rollback.deploymentId);
|
|
expect(await readFile(path, 'utf8')).toBe('original configuration');
|
|
}
|
|
);
|
|
});
|