Defer server file changes until guarded restart activation

Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
This commit is contained in:
2026-09-27 23:06:16 -04:00
parent eb2a852fdc
commit fb48f64de1
36 changed files with 1634 additions and 421 deletions
+94 -29
View File
@@ -6,6 +6,28 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
async function applySource(kind: 'campaign' | 'beta', id: string) {
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const table = kind === 'campaign' ? 'deployment_servers' : 'beta_servers';
const key = kind === 'campaign' ? 'deployment_id' : 'beta_id';
const servers = db().prepare(`SELECT server FROM ${table} WHERE ${key} = ?`).all(id) as { server: string }[];
for (const { server } of servers) {
const operation = requestRestart(server, { kind, id });
applyStagedFilesForOperation(operation.id, server);
}
}
// Scanner tests exercise the queued file application separately from the
// Provisioning occupancy/concurrency integration tests.
async function applyDriftDeployment(input: import('../../src/lib/server/fleet/drift.js').RunDriftInput = {}) {
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const result = await runDriftDeployment(input);
if (result.deploymentId) await applySource('campaign', result.deploymentId);
return result;
}
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(['t5_zombie_theater_1']);
@@ -22,27 +44,67 @@ afterEach(async () => {
});
describe('reviewed drift plans', () => {
it('keeps live deletions, replacements, and inventory unchanged until activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'managed plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
targetRelPath: 'managed.dll', content: Buffer.from('new plugin') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const target = join(destOfKind(server, 'plugin'), 'managed.dll');
const stray = join(destOfKind(server, 'config'), 'stray.cfg');
await writeFile(target, 'current plugin');
await writeFile(stray, 'current settings');
const result = await runDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 1, totalRestored: 1 });
expect(await readFile(target, 'utf8')).toBe('current plugin');
expect(await readFile(stray, 'utf8')).toBe('current settings');
expect(getServerScript(server, 'managed.dll')).toBeUndefined();
await applySource('campaign', result.deploymentId!);
expect(await readFile(target, 'utf8')).toBe('new plugin');
expect(existsSync(stray)).toBe(false);
expect(getServerScript(server, 'managed.dll')?.script_version_id).toBe(version.id);
});
it('preserves a file changed while its reviewed deletion waits for activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'reviewed');
const plan = await previewDrift(['t5_zombie_theater_1']);
const result = await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await writeFile(path, 'new operator settings');
await expect(applySource('campaign', result.deploymentId!)).rejects.toThrow();
expect(await readFile(path, 'utf8')).toBe('new operator settings');
});
it('rejects changed file bytes before any deletion and returns a fresh review', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'original');
const plan = await previewDrift(['t5_zombie_theater_1']);
await writeFile(path, 'changed');
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('changed');
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
const refreshed = await previewDrift(plan.servers);
expect(refreshed.fingerprint).not.toBe(plan.fingerprint);
expect((await runDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
expect((await applyDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
});
it('rejects changed replacement bytes and preserves the reviewed existing file', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const { script, version } = createScript({ name: 'Managed config', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
@@ -51,13 +113,13 @@ describe('reviewed drift plans', () => {
await writeFile(path, 'existing settings');
const plan = await previewDrift(['t5_zombie_theater_1']);
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id);
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('existing settings');
});
it('keeps selected reconciliation within its reviewed server identities', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const second = join(fixture.t5, 't5_zombie_theater_2', 'configs');
await mkdir(second, { recursive: true });
await seedServerRegistry();
@@ -65,8 +127,8 @@ describe('reviewed drift plans', () => {
await writeFile(firstPath, 'one');
await writeFile(join(second, 'stray.cfg'), 'two');
const plan = await previewDrift(['t5_zombie_theater_1']);
await expect(runDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await expect(applyDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
expect(existsSync(firstPath)).toBe(false);
expect(await readFile(join(second, 'stray.cfg'), 'utf8')).toBe('two');
});
@@ -78,7 +140,7 @@ describe('drift scanning', () => {
] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -87,11 +149,11 @@ describe('drift scanning', () => {
const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll';
const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null });
expect(getServerScript(server, relPath)).toBeDefined();
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false);
expect(getServerScript(server, relPath)).toBeUndefined();
});
@@ -99,7 +161,7 @@ describe('drift scanning', () => {
it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -109,13 +171,13 @@ describe('drift scanning', () => {
const content = Buffer.from('promoted content');
const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const inventory = getServerScript(server, relPath);
const path = join(destOfKind(server, kind), relPath);
await writeFile(path, 'changed outside Electric Cherry');
expect(getServerScript(server, relPath)).toEqual(inventory);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(content);
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -123,18 +185,18 @@ describe('drift scanning', () => {
it('compares composed config with disk even when inventory still has the promoted hash', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'changed outside Electric Cherry');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
});
@@ -142,7 +204,7 @@ describe('drift scanning', () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
@@ -150,13 +212,14 @@ describe('drift scanning', () => {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n'));
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false });
await applySource('beta', String(betaId));
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
const betaBytes = await readFile(path);
const stray = join(fixture.t5, server, 'configs', 'stray.cfg');
await writeFile(stray, 'also deferred');
expect(await scanServerDrift(server)).toEqual([]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await readFile(path)).toEqual(betaBytes);
expect(await readFile(stray, 'utf8')).toBe('also deferred');
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
@@ -165,7 +228,7 @@ describe('drift scanning', () => {
it('reports an unavailable registered directory without hiding drift on another game root', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const other = 't6_zombie_tranzit_1';
await mkdir(join(fixture.t6, other, 'configs'), { recursive: true });
@@ -173,7 +236,7 @@ describe('drift scanning', () => {
await seedServerRegistry();
await rm(fixture.t5, { recursive: true });
await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable');
await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
await expect(applyDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
expect(existsSync(fixture.t5)).toBe(false);
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]);
@@ -182,7 +245,7 @@ describe('drift scanning', () => {
it('replaces a seeded config once instead of planning both removal and installation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
@@ -193,7 +256,7 @@ describe('drift scanning', () => {
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'seeded config');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind: 'config', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -201,7 +264,7 @@ describe('drift scanning', () => {
it.each(['identical', 'different'])('reconciles an existing %s scoped file in one pass with an update audit', async (mode) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -215,17 +278,17 @@ describe('drift scanning', () => {
const target = join(destOfKind(server, 'script'), 'zm_spawn_fix.gsc');
await writeFile(target, before);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'zm_spawn_fix.gsc', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await applyDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await readFile(target)).toEqual(before);
expect(getServerScript(server, 'zm_spawn_fix.gsc')).toBeUndefined();
const result = await runDriftDeployment({ servers: [server] });
const result = await applyDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(target)).toEqual(content);
expect(getServerScript(server, 'zm_spawn_fix.gsc')?.script_version_id).toBe(version.id);
expect(db().prepare('SELECT verb, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId))
.toEqual([{ verb: 'updated', content_before: before, content_after: content }]);
expect(await scanServerDrift(server)).toEqual([]);
expect((await runDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
expect((await applyDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
});
it('detects untracked files across script, config, and plugin destinations', async () => {
@@ -335,6 +398,7 @@ async function promoteAndOrphan(
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);
@@ -352,7 +416,7 @@ describe('missing artifact detection', () => {
const { openBeta, promoteBeta } = await import(
'../../src/lib/server/fleet/beta.js'
);
const { scanServerDrift, runDriftDeployment } = await import(
const { scanServerDrift } = await import(
'../../src/lib/server/fleet/drift.js'
);
const { db } = await import('../../src/lib/server/db/index.js');
@@ -383,7 +447,7 @@ describe('missing artifact detection', () => {
expect(missing.length).toBe(1);
expect(missing[0].relPath).toBe('hud.gsc');
const result = await runDriftDeployment({
const result = await applyDriftDeployment({
servers: ['t5_zombie_theater_1'],
dryRun: false,
triggeredBy: 'ui'
@@ -420,6 +484,7 @@ describe('missing artifact detection', () => {
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);