josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Defer server file changes until guarded restart activation

Browse Source 
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
This commit is contained in:
josh
2026-09-27 23:06:16 -04:00
parent eb2a852fdc
commit fb48f64de1
36 changed files with 1634 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/beta-plans.test.ts
+10 -2
View File
@@ -84,9 +84,11 @@ describe('durable beta test plans', () => {
await tick(); await tick();
expect(existsSync(scriptPath(names[1]))).toBe(false);
for (let i = 0; i < 4; i++) await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
expect(existsSync(scriptPath(names[1]))).toBe(false);
expect(getBeta(id)?.state).toBe('running');
expect(listBetaServers(id)[1].stage).toBe('awaiting-empty');
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
for (let i = 0; i < 6; i++) await tick();
expect(getBeta(id)?.state).toBe('ready-promote');
});
@@ -120,6 +122,9 @@ describe('durable beta test plans', () => {
expect(testView(id)?.servers[0].progress[0].current).toBe(25);
expect(testView(id)?.evidence.filter(e => e.kind === 'round_change')).toHaveLength(1);
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(false);
await postTelemetry(gameEvent('heartbeat', 99, 0, names[1]));
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
});
@@ -170,9 +175,12 @@ describe('durable beta test plans', () => {
const { getBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(getBeta(id)?.state).toBe('rolled-back');
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('main() {}');
expect(existsSync(scriptPath(names[1]))).toBe(false);
expect(db().prepare("SELECT server, restart_state FROM deployment_servers").all()).toEqual([{ server: names[0], restart_state: 'pending' }]);
const { sweep } = await import('../helpers/provisioning.js');
await sweep();
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
});
it('keeps snapshots and holds after a paused failure, without deploying subsequent waves', async () => {
tests/unit/beta-promotion.test.ts
+4
View File
@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
@@ -22,6 +23,7 @@ async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'ma
scopeType: scope, scopeValue: scope === 'game' ? 't5' : 'zombie_theater', content: Buffer.from(kind === 'script' ? 'main(){}' : 'set sv_voice "1"\n') });
const version = getLatestVersion(script.id)!;
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [servers[0]], waitForHealth: false, waitForMatch: false });
await activateFiles('beta', betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
return { betaId, script, version };
@@ -40,6 +42,8 @@ describe('reviewed beta promotion', () => {
const result = await promoteBeta(betaId, preview.token);
expect(result).toMatchObject({ awaitingActivation: 1, failedServers: [] });
expect(result.deploymentId).toContain('promote');
expect(existsSync(destination())).toBe(false);
await activateFiles('campaign', result.deploymentId);
expect(await readFile(destination(), 'utf8')).toBe('main(){}');
});
it('rejects changed scope membership before creating a campaign or writing extra files', async () => {
tests/unit/beta-staging.test.ts
+232
View File
@@ -0,0 +1,232 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(servers);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function open(kind: 'script' | 'plugin' | 'baseline' = 'script') {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { script, version } = createScript({ name: `Staged ${kind}`, kind,
targetRelPath: kind === 'script' ? 'nested/test.gsc' : kind === 'plugin' ? 'test.dll' : undefined,
scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'baseline' ? 'set sv_voice "1"\n' : 'beta bytes') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
return { betaId, script, version };
}
describe('staged beta files', () => {
it.each(['script', 'plugin', 'baseline'] as const)('queues %s bytes without creating live directories or recording deployed inventory', async kind => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const dest = destOfKind(servers[0], kind);
await rm(dest, { recursive: true, force: true });
const { betaId } = await open(kind);
const { db } = await import('../../src/lib/server/db/index.js');
expect(existsSync(dest)).toBe(false);
expect(db().prepare('SELECT stage, deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
.toEqual({ stage: 'awaiting-empty', deployed_at: null });
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
expect(db().prepare('SELECT * FROM beta_file_changes').all()).toHaveLength(0);
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
await activateFiles('beta', betaId);
expect(existsSync(dest)).toBe(true);
expect(db().prepare('SELECT deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
.toEqual({ deployed_at: expect.any(String) });
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(1);
});
it('cancels an unactivated beta without changing existing files or queuing a restart', async () => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'original');
const { betaId } = await open();
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(await rollbackBeta(betaId)).toMatchObject({ deploymentId: null, restored: [], dropped: [] });
expect(await readFile(path, 'utf8')).toBe('original');
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ state: 'cancelled' })]);
});
it('captures the actual activation before image and queues an unplanned beta rollback', async () => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'request-time bytes');
const { betaId } = await open();
await writeFile(path, 'activation-time bytes');
await activateFiles('beta', betaId);
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await rollbackBeta(betaId);
expect(result).toMatchObject({ deploymentId: expect.any(String), restored: [servers[0]] });
expect(await readFile(path, 'utf8')).toBe('beta bytes');
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
await activateFiles('campaign', result.deploymentId!);
expect(await readFile(path, 'utf8')).toBe('activation-time bytes');
});
it('holds manual rollback files until its campaign is resumed', async () => {
const { betaId } = await open();
await activateFiles('beta', betaId);
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await rollbackBeta(betaId, { restart: 'manual' });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(result.deploymentId))
.toEqual({ restart_state: 'held' });
await expect(activateFiles('campaign', result.deploymentId!)).rejects.toThrow('no longer eligible');
const { destOfKind } = await import('../../src/lib/server/config.js');
expect(await readFile(join(destOfKind(servers[0], 'script'), 'nested/test.gsc'), 'utf8')).toBe('beta bytes');
});
it.each([false, true])('prepares missing files before retrying a beta (test plan: %s)', async planned => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
const { script, version } = createScript({ name: 'Needs baseline', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false, ...(planned ? { testPlan: defaultPlan() } : {}) });
expect(listBetaServers(betaId)[0].stage).toBe('failed');
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
const retry = () => POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
}) });
expect((await retry()).status).toBe(planned ? 400 : 409);
expect(listBetaServers(betaId)[0].stage).toBe('failed');
const baseline = createScript({ name: 'Required baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id);
expect((await retry()).status).toBe(200);
expect(listBetaServers(betaId)[0]).toMatchObject({ stage: 'awaiting-empty', deployed_at: null });
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ kind: 'config', state: 'pending' })]);
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
});
it.each([false, true])('rejects an interrupted file write on retry (test plan: %s)', async planned => {
const { betaId } = await open();
const { db } = await import('../../src/lib/server/db/index.js');
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
if (planned) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(defaultPlan()));
db().prepare("UPDATE beta_servers SET stage = 'failed' WHERE beta_id = ?").run(betaId);
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId));
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
const result = await POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
}) });
expect(result.status).toBe(planned ? 400 : 409);
expect((await result.json()).error).toContain('file write was interrupted');
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'failed' });
});
it.each([
{ operationState: null, fileState: 'pending', restartState: 'pending', blocked: true },
{ operationState: 'waiting', fileState: 'pending', restartState: 'pending', blocked: true },
{ operationState: 'failed', fileState: 'pending', restartState: 'pending', blocked: false },
{ operationState: 'cancelled', fileState: 'pending', restartState: 'pending', blocked: false },
{ operationState: 'interrupted', fileState: 'applying', restartState: 'failed', blocked: true },
{ operationState: 'cancelled', fileState: 'pending', restartState: 'held', blocked: true },
{ operationState: 'failed', fileState: 'pending', restartState: 'manual', blocked: true },
{ operationState: null, fileState: null, restartState: 'pending', blocked: false }
])('protects new beta admission from queued campaign work: %j', async ({ operationState, fileState, restartState, blocked }) => {
const { betaId, script } = await open();
await activateFiles('beta', betaId);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const { promoteBeta, openBeta } = await import('../../src/lib/server/fleet/beta.js');
const promotion = await promoteBeta(betaId);
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ? AND server = ?')
.run(restartState, promotion.deploymentId, servers[1]);
if (operationState) {
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
const operation = requestRestart(servers[1], { kind: 'campaign', id: promotion.deploymentId });
db().prepare('UPDATE server_deployments SET state = ? WHERE id = ?').run(operationState, operation.id);
}
if (fileState) db().prepare('UPDATE staged_file_changes SET state = ? WHERE source_kind = ? AND source_id = ?')
.run(fileState, 'campaign', promotion.deploymentId);
else db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").run(promotion.deploymentId);
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const newer = uploadNewVersion(script.id, Buffer.from('newer beta bytes'));
const request = openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
if (blocked) await expect(request).rejects.toMatchObject({ kind: 'conflict', details: [expect.objectContaining({ deploymentId: promotion.deploymentId })] });
else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
});
it.each(['script', 'baseline'] as const)('keeps the newest %s promotion when another campaign is queued during its beta', async kind => {
const { betaId: olderBeta, script } = await open(kind);
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const newestBytes = kind === 'script' ? 'newest tested bytes' : 'set sv_voice "0"\n';
const newer = uploadNewVersion(script.id, Buffer.from(newestBytes));
const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { betaId: newerBeta } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
await activateFiles('beta', olderBeta);
await activateFiles('beta', newerBeta);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id IN (?, ?)").run(olderBeta, newerBeta);
const older = await promoteBeta(olderBeta);
const latest = await promoteBeta(newerBeta);
expect(latest.awaitingActivation).toBe(2);
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(stagedFileChanges('campaign', latest.deploymentId, servers[1]))
.toEqual([expect.objectContaining({ preserve_before_image: 1, state: 'pending' })]);
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId });
const second = requestRestart(servers[1], { kind: 'campaign', id: latest.deploymentId });
expect(second.id).toBe(first.id);
await activateFiles('campaign', latest.deploymentId);
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]);
expect(await readFile(join(destOfKind(servers[1], kind), relPath), 'utf8')).toContain(newestBytes.trim());
expect(db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
.get(latest.deploymentId, servers[1])).toEqual({ content_before: null });
});
it.each([false, true])('retains the participant pre-beta image when changed composition activation is %s', async activatePromotion => {
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'config'), configFileOf(servers[0]));
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'original configuration');
const { betaId } = await open('baseline');
await activateFiles('beta', betaId);
const betaBytes = await readFile(path, 'utf8');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { script, version } = createScript({ name: 'Later overlay', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await promoteBeta(betaId);
expect(result.awaitingActivation).toBe(2);
expect(await readFile(path, 'utf8')).toBe(betaBytes);
if (activatePromotion) {
await activateFiles('campaign', result.deploymentId);
expect(await readFile(path, 'utf8')).toContain('set sv_voice "0"');
}
const history = db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
.get(result.deploymentId, servers[0]) as { content_before: Buffer };
expect(history.content_before.toString()).toBe('original configuration');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const rollback = await rollbackPromoteDeployment(result.deploymentId);
expect(rollback.affectedServers).toContain(servers[0]);
expect(await readFile(path, 'utf8')).not.toBe('original configuration');
await activateFiles('campaign', rollback.deploymentId);
expect(await readFile(path, 'utf8')).toBe('original configuration');
});
});
tests/unit/campaign-resume.test.ts
+112
View File
@@ -0,0 +1,112 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
let fixture: Fixture;
const server = 't5_zombie_theater_1';
const campaign = 'held-rollback';
const before = Buffer.from('current live bytes');
const restored = Buffer.from('restored version');
const destination = () => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc');
beforeEach(async () => {
vi.resetModules();
vi.clearAllMocks();
fixture = await newFixture([server]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function heldRollback(state = 'held') {
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
await writeFile(destination(), before);
const now = new Date().toISOString();
db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)")
.run(campaign, now, now);
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error)
VALUES (?, ?, 1, ?, 'Waiting for manual activation')`).run(campaign, server, state);
queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: 'rollback.gsc',
content: restored, inventorySource: 'rollback:beta-1' });
}
async function resume(id = campaign) {
const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
return POST({ params: { id } });
}
describe('campaign resume API', () => {
it.each(['held', 'manual'])('queues %s rollback changes without touching live files', async (state) => {
await heldRollback(state);
const { db } = await import('../../src/lib/server/db/index.js');
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
const response = await resume();
expect(response.status).toBe(202);
expect(await response.json()).toEqual({ ok: true });
expect(db().prepare('SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ restart_state: 'pending', error: null, provisioning_id: null });
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)).toEqual({ finished_at: null });
expect(db().prepare('SELECT state, content FROM staged_file_changes WHERE source_id = ?').get(campaign))
.toEqual({ state: 'pending', content: restored });
expect(await readFile(destination())).toEqual(before);
expect(kick).toHaveBeenCalledTimes(1);
expect((await resume()).status).toBe(409);
expect(kick).toHaveBeenCalledTimes(1);
});
it.each(['cancelled', 'failed', 'interrupted'])('releases an old %s operation so the worker creates fresh restart intent', async (state) => {
await heldRollback();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { db } = await import('../../src/lib/server/db/index.js');
const old = requestRestart(server, { kind: 'campaign', id: campaign });
db().prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
.run(state, new Date().toISOString(), old.id);
expect((await resume()).status).toBe(202);
expect(db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ provisioning_id: null });
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
await sweep();
const next = requestRestart(server, { kind: 'campaign', id: campaign });
expect(next.id).not.toBe(old.id);
expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)).toEqual({ state });
expect(db().prepare('SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid').all(campaign))
.toEqual([{ operation_id: old.id }, { operation_id: next.id }]);
expect(await readFile(destination())).toEqual(before);
});
it('retains an existing waiting operation when its held campaign resumes', async () => {
await heldRollback();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const old = requestRestart(server, { kind: 'campaign', id: campaign });
expect((await resume()).status).toBe(202);
expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id);
});
it.each(['missing', 'done', 'deleted'])('rejects a campaign with %s changes that cannot resume', async (condition) => {
const { db } = await import('../../src/lib/server/db/index.js');
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held');
if (condition === 'deleted') {
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry(); // Reusing its name must not resume old work.
}
const response = await resume();
expect(response.status).toBe(409);
expect(await response.json()).toEqual({ error: 'This campaign has no held changes to resume.' });
expect(kick).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
if (condition !== 'missing') {
expect(await readFile(destination())).toEqual(before);
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ restart_state: condition === 'done' ? 'done' : 'held' });
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign))
.toMatchObject({ finished_at: expect.any(String) });
}
});
});
tests/unit/config-flow.test.ts
+11 -4
View File
@@ -3,6 +3,7 @@ import { readFile, mkdir } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
@@ -34,7 +35,7 @@ async function seedFleet(): Promise<void> {
}
describe('config baseline + overlay flow', () => {
it('composes baseline + overlay and writes dedicated_sp.cfg on beta open, restores pre-beta state on rollback', async () => {
it('composes baseline + overlay for activation and queues restoration of the pre-beta state', async () => {
await seedFleet();
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
@@ -70,9 +71,10 @@ describe('config baseline + overlay flow', () => {
waitForMatch: false
});
// 4. Verify: composed cfg lands at configs/dedicated_sp.cfg (in place,
// NO -beta suffix — the game reads the exact filename).
// The live filename is untouched until activation.
const cfgPath = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg');
expect(existsSync(cfgPath)).toBe(false);
await activateFiles('beta', betaId);
expect(existsSync(cfgPath)).toBe(true);
const composed = (await readFile(cfgPath)).toString('utf8');
expect(composed).toContain('set g_inactivity "0"');
@@ -82,7 +84,9 @@ describe('config baseline + overlay flow', () => {
// 5. Rollback restores the pre-beta state. There was no cfg on disk
// beforehand, so rollback should just remove the beta-written file.
await rollbackBeta(betaId);
const rollback = await rollbackBeta(betaId);
expect(await readFile(cfgPath, 'utf8')).toBe(composed);
await activateFiles('campaign', rollback.deploymentId!);
expect(existsSync(cfgPath)).toBe(false);
});
@@ -110,6 +114,7 @@ describe('config baseline + overlay flow', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
const row = db()
.prepare(`SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?`)
@@ -146,6 +151,8 @@ describe('plugin kind writes to plugin destination', () => {
});
const pluginPath = join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/plugins', 'demo.dll');
expect(existsSync(pluginPath)).toBe(false);
await activateFiles('beta', betaId);
expect(existsSync(pluginPath)).toBe(true);
const bytes = await readFile(pluginPath);
expect(bytes.equals(Buffer.from([0x00, 0x01, 0x02, 0x03]))).toBe(true);
tests/unit/create-server.test.ts
+5 -1
View File
@@ -645,7 +645,11 @@ describe('server creation map selection', () => {
expect(await scanServerDrift(serverName)).toEqual([]);
const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg');
await rm(mapPath);
expect((await runDriftDeployment({ servers: [serverName] })).totalRestored).toBe(1);
const repair = await runDriftDeployment({ servers: [serverName] });
expect(repair.totalRestored).toBe(1);
expect(existsSync(mapPath)).toBe(false);
const { activateFiles } = await import('../helpers/staged-files.js');
await activateFiles('campaign', repair.deploymentId!);
expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(await scanServerDrift(serverName)).toEqual([]);
});
tests/unit/deleted-server-guards.test.ts
+6 -1
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
import { gameEvent } from '../helpers/telemetry.js';
@@ -107,6 +108,7 @@ describe('deleted server operational guards', () => {
it('skips deleted campaign targets during restart and rollback while restoring live targets', async () => {
for (const name of names) await writeFile(path(name), 'original content');
const { betaId } = await open();
await activateFiles('beta', betaId);
const { db } = await import('../../src/lib/server/db/index.js');
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
@@ -115,12 +117,15 @@ describe('deleted server operational guards', () => {
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const campaign = await promoteBeta(betaId);
const provisioningBefore = db().prepare('SELECT id FROM server_deployments').all();
await replace(names[1]);
await sweep();
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual(provisioningBefore);
expect(() => requestRestart(names[1], { kind: 'campaign', id: campaign.deploymentId })).toThrow('Restart source no longer exists');
const result = await rollbackPromoteDeployment(campaign.deploymentId);
expect(result.affectedServers).toEqual([names[0]]);
expect(await readFile(path(names[0]), 'utf8')).toBe('beta content');
await activateFiles('campaign', result.deploymentId);
expect(await readFile(path(names[0]), 'utf8')).toBe('original content');
expect(await readFile(path(names[1]), 'utf8')).toBe('replacement content');
expect(db().prepare('SELECT server FROM deployment_servers WHERE deployment_id = ?').all(result.deploymentId)).toEqual([{ server: names[0] }]);
tests/unit/drift-kinds.test.ts
+94 -29
View File
@@ -6,6 +6,28 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
async function applySource(kind: 'campaign' | 'beta', id: string) {
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const table = kind === 'campaign' ? 'deployment_servers' : 'beta_servers';
const key = kind === 'campaign' ? 'deployment_id' : 'beta_id';
const servers = db().prepare(`SELECT server FROM ${table} WHERE ${key} = ?`).all(id) as { server: string }[];
for (const { server } of servers) {
const operation = requestRestart(server, { kind, id });
applyStagedFilesForOperation(operation.id, server);
}
}
// Scanner tests exercise the queued file application separately from the
// Provisioning occupancy/concurrency integration tests.
async function applyDriftDeployment(input: import('../../src/lib/server/fleet/drift.js').RunDriftInput = {}) {
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const result = await runDriftDeployment(input);
if (result.deploymentId) await applySource('campaign', result.deploymentId);
return result;
}
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(['t5_zombie_theater_1']);
@@ -22,27 +44,67 @@ afterEach(async () => {
});
describe('reviewed drift plans', () => {
it('keeps live deletions, replacements, and inventory unchanged until activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'managed plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
targetRelPath: 'managed.dll', content: Buffer.from('new plugin') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const target = join(destOfKind(server, 'plugin'), 'managed.dll');
const stray = join(destOfKind(server, 'config'), 'stray.cfg');
await writeFile(target, 'current plugin');
await writeFile(stray, 'current settings');
const result = await runDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 1, totalRestored: 1 });
expect(await readFile(target, 'utf8')).toBe('current plugin');
expect(await readFile(stray, 'utf8')).toBe('current settings');
expect(getServerScript(server, 'managed.dll')).toBeUndefined();
await applySource('campaign', result.deploymentId!);
expect(await readFile(target, 'utf8')).toBe('new plugin');
expect(existsSync(stray)).toBe(false);
expect(getServerScript(server, 'managed.dll')?.script_version_id).toBe(version.id);
});
it('preserves a file changed while its reviewed deletion waits for activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'reviewed');
const plan = await previewDrift(['t5_zombie_theater_1']);
const result = await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await writeFile(path, 'new operator settings');
await expect(applySource('campaign', result.deploymentId!)).rejects.toThrow();
expect(await readFile(path, 'utf8')).toBe('new operator settings');
});
it('rejects changed file bytes before any deletion and returns a fresh review', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'original');
const plan = await previewDrift(['t5_zombie_theater_1']);
await writeFile(path, 'changed');
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('changed');
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
const refreshed = await previewDrift(plan.servers);
expect(refreshed.fingerprint).not.toBe(plan.fingerprint);
expect((await runDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
expect((await applyDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
});
it('rejects changed replacement bytes and preserves the reviewed existing file', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const { script, version } = createScript({ name: 'Managed config', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
@@ -51,13 +113,13 @@ describe('reviewed drift plans', () => {
await writeFile(path, 'existing settings');
const plan = await previewDrift(['t5_zombie_theater_1']);
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id);
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('existing settings');
});
it('keeps selected reconciliation within its reviewed server identities', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const second = join(fixture.t5, 't5_zombie_theater_2', 'configs');
await mkdir(second, { recursive: true });
await seedServerRegistry();
@@ -65,8 +127,8 @@ describe('reviewed drift plans', () => {
await writeFile(firstPath, 'one');
await writeFile(join(second, 'stray.cfg'), 'two');
const plan = await previewDrift(['t5_zombie_theater_1']);
await expect(runDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await expect(applyDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
expect(existsSync(firstPath)).toBe(false);
expect(await readFile(join(second, 'stray.cfg'), 'utf8')).toBe('two');
});
@@ -78,7 +140,7 @@ describe('drift scanning', () => {
] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -87,11 +149,11 @@ describe('drift scanning', () => {
const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll';
const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null });
expect(getServerScript(server, relPath)).toBeDefined();
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false);
expect(getServerScript(server, relPath)).toBeUndefined();
});
@@ -99,7 +161,7 @@ describe('drift scanning', () => {
it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -109,13 +171,13 @@ describe('drift scanning', () => {
const content = Buffer.from('promoted content');
const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const inventory = getServerScript(server, relPath);
const path = join(destOfKind(server, kind), relPath);
await writeFile(path, 'changed outside Electric Cherry');
expect(getServerScript(server, relPath)).toEqual(inventory);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(content);
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -123,18 +185,18 @@ describe('drift scanning', () => {
it('compares composed config with disk even when inventory still has the promoted hash', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'changed outside Electric Cherry');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
});
@@ -142,7 +204,7 @@ describe('drift scanning', () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
@@ -150,13 +212,14 @@ describe('drift scanning', () => {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n'));
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false });
await applySource('beta', String(betaId));
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
const betaBytes = await readFile(path);
const stray = join(fixture.t5, server, 'configs', 'stray.cfg');
await writeFile(stray, 'also deferred');
expect(await scanServerDrift(server)).toEqual([]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await readFile(path)).toEqual(betaBytes);
expect(await readFile(stray, 'utf8')).toBe('also deferred');
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
@@ -165,7 +228,7 @@ describe('drift scanning', () => {
it('reports an unavailable registered directory without hiding drift on another game root', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const other = 't6_zombie_tranzit_1';
await mkdir(join(fixture.t6, other, 'configs'), { recursive: true });
@@ -173,7 +236,7 @@ describe('drift scanning', () => {
await seedServerRegistry();
await rm(fixture.t5, { recursive: true });
await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable');
await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
await expect(applyDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
expect(existsSync(fixture.t5)).toBe(false);
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]);
@@ -182,7 +245,7 @@ describe('drift scanning', () => {
it('replaces a seeded config once instead of planning both removal and installation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
@@ -193,7 +256,7 @@ describe('drift scanning', () => {
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'seeded config');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind: 'config', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -201,7 +264,7 @@ describe('drift scanning', () => {
it.each(['identical', 'different'])('reconciles an existing %s scoped file in one pass with an update audit', async (mode) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -215,17 +278,17 @@ describe('drift scanning', () => {
const target = join(destOfKind(server, 'script'), 'zm_spawn_fix.gsc');
await writeFile(target, before);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'zm_spawn_fix.gsc', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await applyDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await readFile(target)).toEqual(before);
expect(getServerScript(server, 'zm_spawn_fix.gsc')).toBeUndefined();
const result = await runDriftDeployment({ servers: [server] });
const result = await applyDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(target)).toEqual(content);
expect(getServerScript(server, 'zm_spawn_fix.gsc')?.script_version_id).toBe(version.id);
expect(db().prepare('SELECT verb, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId))
.toEqual([{ verb: 'updated', content_before: before, content_after: content }]);
expect(await scanServerDrift(server)).toEqual([]);
expect((await runDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
expect((await applyDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
});
it('detects untracked files across script, config, and plugin destinations', async () => {
@@ -335,6 +398,7 @@ async function promoteAndOrphan(
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);
@@ -352,7 +416,7 @@ describe('missing artifact detection', () => {
const { openBeta, promoteBeta } = await import(
'../../src/lib/server/fleet/beta.js'
);
const { scanServerDrift, runDriftDeployment } = await import(
const { scanServerDrift } = await import(
'../../src/lib/server/fleet/drift.js'
);
const { db } = await import('../../src/lib/server/db/index.js');
@@ -383,7 +447,7 @@ describe('missing artifact detection', () => {
expect(missing.length).toBe(1);
expect(missing[0].relPath).toBe('hud.gsc');
const result = await runDriftDeployment({
const result = await applyDriftDeployment({
servers: ['t5_zombie_theater_1'],
dryRun: false,
triggeredBy: 'ui'
@@ -420,6 +484,7 @@ describe('missing artifact detection', () => {
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);
tests/unit/map-config-flow.test.ts
+10 -2
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
@@ -37,12 +38,14 @@ describe('map selection through config lifecycle', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
expect(await readFile(cfgPath(servers[0]), 'utf8')).toContain('set sv_maprotation "map zombie_theater"');
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?")
.run(new Date().toISOString(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const promoted = await promoteBeta(betaId);
expect(new Set(promoted.landed)).toEqual(new Set(servers));
await activateFiles('campaign', promoted.deploymentId);
const ascension = await readFile(cfgPath(servers[1]), 'utf8');
expect(ascension).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(ascension).toContain('set sv_voice "1"');
@@ -51,6 +54,7 @@ describe('map selection through config lifecycle', () => {
await rm(cfgPath(servers[1]));
const repaired = await runDriftDeployment({ servers: [servers[1]] });
expect(repaired.totalRestored).toBe(1);
await activateFiles('campaign', repaired.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(ascension);
expect(await scanServerDrift(servers[1])).toEqual([]);
});
@@ -75,11 +79,14 @@ describe('map selection through config lifecycle', () => {
scripts: [{ scriptId: overlay.id, scriptVersionId: getLatestVersion(overlay.id)!.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
const preview = await readFile(cfgPath(servers[1]), 'utf8');
expect(preview).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(preview).toContain('set g_inactivity "0"');
expect(preview).not.toContain('map zombie_moon');
await rollbackBeta(betaId);
const rollback = await rollbackBeta(betaId);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(preview);
await activateFiles('campaign', rollback.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(oldConfig);
});
@@ -90,7 +97,8 @@ describe('map selection through config lifecycle', () => {
await writeFile(join(fixture.t5, servers[1], 'configs', 'stray.cfg'), 'set stray "1"');
const drift = await scanServerDrift(servers[1]);
expect(drift.map((entry) => entry.relPath)).toEqual(['stray.cfg']);
await runDriftDeployment({ servers: [servers[1]] });
const driftResult = await runDriftDeployment({ servers: [servers[1]] });
await activateFiles('campaign', driftResult.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(defaults);
});
});
tests/unit/operation-history.test.ts
+1 -1
View File
@@ -51,7 +51,7 @@ describe('operation progress', () => {
const server = { stage: 'awaiting-empty', deployed_at: 'now', restarted_at: null, healthy_at: null, match_seen_at: null };
it('uses the slowest beta server and omits disabled gates', () => {
const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [server, { ...server, stage: 'ready' }]);
expect(flow.map(s => s.label)).toEqual(['Deploy artifacts', 'Wait for empty servers', 'Restart servers', 'Review and promote']);
expect(flow.map(s => s.label)).toEqual(['Stage file changes', 'Wait for empty servers', 'Apply files and restart', 'Review and promote']);
expect(flow.map(s => s.state)).toEqual(['done', 'current', 'pending', 'pending']);
});
it('does not mark a failed deployment as completed', () => {
tests/unit/rollback-staging.test.ts
+128
View File
@@ -0,0 +1,128 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const server = 't5_zombie_theater_1';
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([server]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function promotion(kind: 'script' | 'plugin' | 'baseline', staged: boolean) {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const before = Buffer.from(kind === 'baseline' ? 'sv_voice 0\n' : 'original');
const after = Buffer.from(kind === 'baseline' ? 'sv_voice 1\n' : 'promoted');
const relPath = kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc';
const { script, version: prior } = createScript({ name: 'managed', kind, targetRelPath: relPath, scopeType: 'game', scopeValue: 't5', content: before });
const version = uploadNewVersion(script.id, after);
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('promote', 'promote', ?, 'ui')").run(new Date().toISOString());
db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('promote', ?, 1, 'pending')").run(server);
db().prepare("INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES ('promote', ?, ?, ?, ?)")
.run(script.id, version.id, prior.id, kind);
const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)`).run(server, relPath, sha256Buffer(before), sha256Buffer(after), before, after);
const path = join(destOfKind(server, kind), relPath);
await mkdir(dirname(path), { recursive: true });
await writeFile(path, staged ? before : after);
if (staged) queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind, relPath, content: after,
scriptVersionId: version.id, inventorySource: 'promote:promote', deploymentFileId: Number(file.lastInsertRowid) });
return { path, before, after, script, prior };
}
async function applyCampaign(id: string) {
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const operation = requestRestart(server, { kind: 'campaign', id });
applyStagedFilesForOperation(operation.id, server);
return operation;
}
describe('deferred promotion rollback', () => {
it('withdraws an unapplied promotion without restoring or restarting its server', async () => {
const { path, before } = await promotion('script', true);
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation, isFileApplied } = await import('../../src/lib/server/fleet/staged-files.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const operation = requestRestart(server, { kind: 'campaign', id: 'promote' });
const rollback = await rollbackPromoteDeployment('promote');
expect(rollback.affectedServers).toEqual([]);
expect(await readFile(path)).toEqual(before);
expect(isFileApplied('campaign', 'promote', server, 'script', 'managed.gsc')).toBe(false);
expect(db().prepare("SELECT restart_state FROM deployment_servers WHERE deployment_id = 'promote'").get()).toEqual({ restart_state: 'skipped' });
expect(db().prepare('SELECT * FROM deployment_servers WHERE deployment_id = ?').all(rollback.deploymentId)).toEqual([]);
applyStagedFilesForOperation(operation.id, server);
expect(await readFile(path)).toEqual(before);
});
it.each(['script', 'plugin', 'baseline'] as const)('queues restoration of applied %s bytes until rollback activation', async (kind) => {
const { path, before, after, prior } = await promotion(kind, true);
await applyCampaign('promote');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const result = await rollbackPromoteDeployment('promote');
expect(result.affectedServers).toEqual([server]);
expect(await readFile(path)).toEqual(after);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
expect(getServerScript(server, kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc')?.script_version_id).toBe(prior.id);
});
it('restores only applied files when a promotion was partly installed', async () => {
const { path, before } = await promotion('script', true);
await applyCampaign('promote');
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const pending = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after)
VALUES ('promote', ?, 'added', 'never.gsc', ?)`).run(server, Buffer.from('queued'));
queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind: 'script', relPath: 'never.gsc',
content: Buffer.from('queued'), inventorySource: 'promote:promote', deploymentFileId: Number(pending.lastInsertRowid) });
const result = await rollbackPromoteDeployment('promote');
expect(db().prepare('SELECT rel_path FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId))
.toEqual([{ rel_path: 'managed.gsc' }]);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
expect(existsSync(join(dirname(path), 'never.gsc'))).toBe(false);
});
it('keeps another campaign eligible when withdrawing a shared activation source', async () => {
const { path, before } = await promotion('script', true);
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('other', 'drift', ?, 'ui')").run(new Date().toISOString());
db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('other', ?, 1, 'pending')").run(server);
queueFileChange({ sourceKind: 'campaign', sourceId: 'other', server, kind: 'script', relPath: 'other.gsc',
content: Buffer.from('other campaign'), inventorySource: 'drift:other' });
const shared = requestRestart(server, { kind: 'campaign', id: 'promote' });
expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).toBe(shared.id);
await rollbackPromoteDeployment('promote');
await applyCampaign('other');
expect(await readFile(path)).toEqual(before);
expect(await readFile(join(dirname(path), 'other.gsc'), 'utf8')).toBe('other campaign');
});
it('stages rollback for legacy promotions with no deferred file records', async () => {
const { path, before, after } = await promotion('plugin', false);
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const result = await rollbackPromoteDeployment('promote');
expect(result.affectedServers).toEqual([server]);
expect(await readFile(path)).toEqual(after);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
});
});
tests/unit/script-flow.test.ts
+11 -1
View File
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
@@ -62,9 +63,12 @@ describe('editing promoted script scopes', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
return promoteBeta(betaId);
const result = await promoteBeta(betaId);
await activateFiles('campaign', result.deploymentId);
return result;
}
await promote();
@@ -105,6 +109,7 @@ describe('editing promoted script scopes', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const inventory = db().prepare('SELECT * FROM server_scripts').all();
expect(inventory.length).toBeGreaterThan(0);
@@ -143,6 +148,7 @@ describe('promote creates a deployment', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Force all beta_servers to 'ready' and beta to 'ready-promote'.
db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`)
@@ -216,6 +222,7 @@ describe('rollback beta on ready-promote marks version test-failed', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Force ready-promote so the state machine gate is closed.
db().prepare(`UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?`).run(betaId);
@@ -253,10 +260,12 @@ describe('rollback promote deployment', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`)
.run(new Date().toISOString(), betaId);
db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId);
const promote = await promoteBeta(betaId);
await activateFiles('campaign', promote.deploymentId);
// Sanity: script is promoted.
expect(getScript(script.id)!.current_version_id).toBe(v.id);
@@ -329,6 +338,7 @@ describe('multi-script beta', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Both scripts are joined to the beta.
const bundle = listBetaScripts(betaId);
tests/unit/staged-files.test.ts
+440
View File
@@ -0,0 +1,440 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { existsSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('node:fs', async (importOriginal) => {
const original = await importOriginal<typeof import('node:fs')>();
return { ...original, writeFileSync: vi.fn(original.writeFileSync) };
});
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
}));
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
const relPath = 'staged.gsc';
const before = Buffer.from('main(){ /* current live script */ }');
const after = Buffer.from('main(){ /* reviewed replacement */ }');
type Source = 'campaign' | 'beta';
const destination = (server = servers[0]) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath);
beforeEach(async () => {
vi.resetModules();
vi.resetAllMocks();
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
vi.mocked(writeFileSync).mockImplementation(actualFs.writeFileSync);
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-09-27T15:00:00Z'));
fixture = await newFixture(servers);
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
vi.stubEnv('KOMODO_API_KEY', 'test');
vi.stubEnv('KOMODO_API_SECRET', 'test');
vi.stubEnv('RCON_PASSWORD', 'test');
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(rconStatusResult).mockResolvedValue({
ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
});
});
afterEach(async () => {
await fixture.cleanup();
vi.unstubAllEnvs();
vi.useRealTimers();
});
async function queueChange(source: Source, targets = [servers[0]]) {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const original = createScript({ name: 'Staged script', kind: 'script', targetRelPath: relPath,
scopeType: 'game', scopeValue: 't5', content: before });
const replacement = uploadNewVersion(original.script.id, after);
for (const server of targets) {
await writeFile(destination(server), before);
recordScript(server, relPath, original.version.id, original.version.sha256, before.length, 'fixture');
}
const inventory = getServerScript(targets[0], relPath);
let sourceId: string;
if (source === 'campaign') {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(replacement.id, original.script.id);
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const result = await runDriftDeployment({ servers: targets });
expect(result.deploymentId).toBeTruthy();
sourceId = result.deploymentId!;
} else {
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await openBeta({ scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }],
servers: targets, waitForHealth: false, waitForMatch: false });
sourceId = String(result.betaId);
}
return { sourceId, inventory, replacement };
}
async function tick(source: Source) {
const { sweep, betaTick } = await import('../helpers/provisioning.js');
await (source === 'campaign' ? sweep() : betaTick());
}
async function expectLiveUnchanged(inventory: Awaited<ReturnType<typeof queueChange>>['inventory'], server = servers[0]) {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(await readFile(destination(server))).toEqual(before);
expect(getServerScript(server, relPath)).toEqual(inventory);
}
const blockers = ['occupied', 'unknown occupancy', 'missing Komodo', 'missing RCON', 'previous restart', 'rollout capacity'] as const;
describe.each<Source>(['campaign', 'beta'])('%s staged file activation', (source) => {
it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => {
const { db } = await import('../../src/lib/server/db/index.js');
const { config } = await import('../../src/lib/server/config.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' });
if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' });
if (blocker === 'previous restart' || blocker === 'rollout capacity') {
db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)')
.run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString());
}
const { inventory, replacement } = await queueChange(source);
await expectLiveUnchanged(inventory);
await tick(source);
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
// The replacement and its inventory must already be installed when
// Komodo sees the activation request, after every admission gate passes.
Object.assign(config.komodo, { address: 'http://komodo.test' });
Object.assign(config.rcon, { password: 'test' });
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
db().prepare('DELETE FROM restart_rollout').run();
let bytesAtRestart: Buffer | undefined;
let inventoryAtRestart: typeof inventory;
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
bytesAtRestart = await readFile(destination());
inventoryAtRestart = getServerScript(servers[0], relPath);
return { ok: true };
});
await tick(source);
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(bytesAtRestart).toEqual(after);
expect(inventoryAtRestart).toMatchObject({
script_version_id: replacement.id, sha256: replacement.sha256, size: after.length
});
});
it('does not write when its provisioning request is cancelled during the occupancy probe', async () => {
const { inventory } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'")
.get(servers[0]) as { id: string };
cancelProvisioning(operation.id);
return 0;
});
await tick(source);
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0]))
.toEqual({ state: 'cancelled' });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('never applies old staged work to a replacement server created during its occupancy probe', async () => {
await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const replacementBytes = Buffer.from('new server instance');
let replacementInventory: ReturnType<typeof getServerScript>;
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
await seedServerRegistry();
await writeFile(destination(), replacementBytes);
recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement');
replacementInventory = getServerScript(servers[0], relPath);
return 0;
});
await tick(source);
expect(await readFile(destination())).toEqual(replacementBytes);
expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('preserves existing file permissions when activating staged bytes', async () => {
await queueChange(source);
await chmod(destination(), 0o640);
await tick(source);
expect(await readFile(destination())).toEqual(after);
expect((await stat(destination())).mode & 0o777).toBe(0o640);
});
it('applies the exact queued bytes even if artifact storage changes while waiting', async () => {
const { replacement } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?')
.run(Buffer.from('changed after review'), replacement.id);
await tick(source);
expect(await readFile(destination())).toEqual(after);
});
it('keeps the queued payload across an application restart', async () => {
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconGetPlayerCount).mockResolvedValue(2);
const { inventory } = await queueChange(source);
await tick(source);
await expectLiveUnchanged(inventory);
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
vi.resetModules();
await tick(source);
await expectLiveUnchanged(inventory);
const freshRcon = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0);
await tick(source);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(await readFile(destination())).toEqual(after);
});
it('does not restart or retain rollout capacity when installing staged files fails', async () => {
const { inventory } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
// A directory appearing at the destination causes a deterministic I/O
// failure even when the test process can bypass ordinary file modes.
await rm(destination());
await mkdir(destination());
await tick(source);
expect(komodoRestart).not.toHaveBeenCalled();
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]);
expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ state: 'failed', error: expect.any(String) });
});
});
describe('staged source eligibility', () => {
it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => {
const { sourceId, inventory } = await queueChange('campaign');
const { db } = await import('../../src/lib/server/db/index.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await sweep();
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId);
await processProvisioning();
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('does not install a beta withdrawn after it requests provisioning', async () => {
const { sourceId, inventory } = await queueChange('beta');
const { db } = await import('../../src/lib/server/db/index.js');
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await betaTick();
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId);
await processProvisioning();
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
});
describe('staged drift removals and review guards', () => {
it('defers removing a file and its inventory until the empty-server restart', async () => {
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await writeFile(destination(), before);
recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture');
const inventory = getServerScript(servers[0], relPath);
vi.mocked(rconGetPlayerCount).mockResolvedValue(1);
await runDriftDeployment({ servers: [servers[0]] });
await expectLiveUnchanged(inventory);
await tick('campaign');
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
let fileExistedAtRestart: boolean | undefined;
let inventoryAtRestart: typeof inventory;
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
fileExistedAtRestart = existsSync(destination());
inventoryAtRestart = getServerScript(servers[0], relPath);
return { ok: true };
});
await tick('campaign');
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(fileExistedAtRestart).toBe(false);
expect(inventoryAtRestart).toBeUndefined();
});
it('fails without overwriting a reviewed target changed while its restart was queued', async () => {
const { inventory } = await queueChange('campaign');
const externalChange = Buffer.from('changed outside Electric Cherry while queued');
await writeFile(destination(), externalChange);
await tick('campaign');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(await readFile(destination())).toEqual(externalChange);
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ error: expect.stringContaining('changed while queued') });
});
});
describe('shared activation and partial writes', () => {
it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => {
const first = await queueChange('campaign');
const { db } = await import('../../src/lib/server/db/index.js');
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const second = await runDriftDeployment({ servers: [servers[0]] });
await sweep();
const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[];
expect(linked).toHaveLength(2);
expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1);
db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId);
await processProvisioning();
await expectLiveUnchanged(first.inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' });
db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId);
await tick('campaign');
expect(await readFile(destination())).toEqual(after);
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
.toEqual([{ state: 'applied' }, { state: 'applied' }]);
});
it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => {
const campaign = await queueChange('campaign');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
await sweep();
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes'));
await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
await expectLiveUnchanged(campaign.inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM betas').all()).toEqual([]);
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId))
.toEqual({ restart_state: 'pending' });
expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId))
.toEqual({ state: 'pending' });
});
it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => {
const beta = await queueChange('beta');
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
// Recreate overlapping work retained from before the admission guard.
// The beta was already open; this older drift still expects its original
// live bytes and must fail independently of the later restoration.
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)")
.run(new Date(Date.now() - 60_000).toISOString());
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]);
queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script',
relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift',
expectedBeforeSha: sha256Buffer(before) });
await sweep();
await tick('beta');
expect(await readFile(destination())).toEqual(after);
const rollback = await rollbackBeta(Number(beta.sourceId));
vi.setSystemTime(Date.now() + 15_000);
await tick('campaign');
expect(await readFile(destination())).toEqual(before);
expect(komodoRestart).toHaveBeenCalledTimes(2);
const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?')
.get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string };
const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?')
.get(rollback.deploymentId) as { provisioning_id: string };
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('changed while queued') });
expect(restoration.provisioning_id).not.toBe(old.provisioning_id);
});
it('records partial installation without restarting and never automatically retries the failed write', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after });
const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after });
const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc');
for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) {
await writeFile(path, before);
recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture');
}
const secondInventory = getServerScript(servers[0], 'second.gsc');
await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })),
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
vi.mocked(writeFileSync).mockImplementation((path, content, options) => {
if (path === secondPath) throw new Error('Simulated disk write failure');
actualFs.writeFileSync(path, content, options);
});
await tick('beta');
expect(await readFile(destination())).toEqual(after);
expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id);
expect(await readFile(secondPath)).toEqual(before);
expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
.toEqual([{ state: 'applied' }, { state: 'applying' }]);
expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get())
.toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
const attempts = vi.mocked(writeFileSync).mock.calls.length;
await tick('beta');
expect(writeFileSync).toHaveBeenCalledTimes(attempts);
expect(komodoRestart).not.toHaveBeenCalled();
});
});
tests/unit/t6-config-flow.test.ts
+15 -3
View File
@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const town = 't6_zombie_town_1';
@@ -39,20 +40,29 @@ describe('T6 config lifecycle in a mixed fleet', () => {
expect(script.target_rel_path).toBe('dedicated_zm.cfg');
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [town], waitForHealth: false, waitForMatch: false });
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
await activateFiles('beta', betaId);
expect(await readFile(cfgPath(town), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const promoted = await promoteBeta(betaId);
expect(new Set(promoted.landed)).toEqual(new Set([town, farm]));
expect(existsSync(cfgPath(farm))).toBe(false);
await activateFiles('campaign', promoted.deploymentId);
const farmConfig = await readFile(cfgPath(farm), 'utf8');
expect(farmConfig).toContain('execgts zm_standard_farm.cfg map zm_transit');
expect(existsSync(join(fixture.t5, t5, 'configs/dedicated_sp.cfg'))).toBe(false);
expect(existsSync(join(fixture.t6, farm, 'configs/dedicated_sp.cfg'))).toBe(false);
expect(await scanServerDrift(farm)).toEqual([]);
await rm(cfgPath(farm));
expect((await runDriftDeployment({ servers: [farm] })).totalRestored).toBe(1);
const drift = await runDriftDeployment({ servers: [farm] });
expect(drift.totalRestored).toBe(1);
expect(existsSync(cfgPath(farm))).toBe(false);
await activateFiles('campaign', drift.deploymentId!);
expect(await readFile(cfgPath(farm), 'utf8')).toBe(farmConfig);
await rollbackPromoteDeployment(promoted.deploymentId);
const rollback = await rollbackPromoteDeployment(promoted.deploymentId);
expect(await readFile(cfgPath(farm), 'utf8')).toBe(farmConfig);
await activateFiles('campaign', rollback.deploymentId);
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
expect(existsSync(cfgPath(farm))).toBe(false);
});
@@ -71,7 +81,9 @@ describe('T6 config lifecycle in a mixed fleet', () => {
content: Buffer.from('sv_voice 0\n') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [town], waitForHealth: false, waitForMatch: false });
await rollbackBeta(betaId);
await activateFiles('beta', betaId);
const rollback = await rollbackBeta(betaId);
await activateFiles('campaign', rollback.deploymentId!);
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
expect(await scanServerDrift(town)).toEqual([]);
await runDriftDeployment({ servers: [town] });