import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_zombie_theater_1']); delete process.env.KOMODO_ADDRESS; delete process.env.RCON_PASSWORD; // Ensure the three per-kind destinations exist under the fixture tree. await mkdir(join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/storage/t5/raw/scripts/sp'), { recursive: true }); await mkdir(join(fixture.t5, 't5_zombie_theater_1', 'configs'), { recursive: true }); await mkdir(join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/plugins'), { recursive: true }); }); afterEach(async () => { await fixture.cleanup(); }); describe('reviewed drift plans', () => { it('rejects changed file bytes before any deletion and returns a fresh review', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); await seedServerRegistry(); const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg'); await writeFile(path, 'original'); const plan = await previewDrift(['t5_zombie_theater_1']); await writeFile(path, 'changed'); await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged); expect(await readFile(path, 'utf8')).toBe('changed'); expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]); const refreshed = await previewDrift(plan.servers); expect(refreshed.fingerprint).not.toBe(plan.fingerprint); expect((await runDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1); }); it('rejects changed replacement bytes and preserves the reviewed existing file', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); await seedServerRegistry(); const { script, version } = createScript({ name: 'Managed config', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg'); await writeFile(path, 'existing settings'); const plan = await previewDrift(['t5_zombie_theater_1']); db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id); await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged); expect(await readFile(path, 'utf8')).toBe('existing settings'); }); it('keeps selected reconciliation within its reviewed server identities', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js'); const second = join(fixture.t5, 't5_zombie_theater_2', 'configs'); await mkdir(second, { recursive: true }); await seedServerRegistry(); const firstPath = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg'); await writeFile(firstPath, 'one'); await writeFile(join(second, 'stray.cfg'), 'two'); const plan = await previewDrift(['t5_zombie_theater_1']); await expect(runDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged); await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint }); expect(existsSync(firstPath)).toBe(false); expect(await readFile(join(second, 'stray.cfg'), 'utf8')).toBe('two'); }); }); describe('drift scanning', () => { it.each([ ['script', 'moved'], ['script', 'unset'], ['plugin', 'moved'], ['plugin', 'unset'] ] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll'; const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); await runDriftDeployment({ servers: [server] }); setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null }); expect(getServerScript(server, relPath)).toBeDefined(); expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]); await runDriftDeployment({ servers: [server] }); expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false); expect(getServerScript(server, relPath)).toBeUndefined(); }); it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const relPath = kind === 'script' ? 'managed.gsc' : 'managed.dll'; const content = Buffer.from('promoted content'); const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); await runDriftDeployment({ servers: [server] }); const inventory = getServerScript(server, relPath); const path = join(destOfKind(server, kind), relPath); await writeFile(path, 'changed outside Electric Cherry'); expect(getServerScript(server, relPath)).toEqual(inventory); expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]); await runDriftDeployment({ servers: [server] }); expect(await readFile(path)).toEqual(content); expect(await scanServerDrift(server)).toEqual([]); }); it('compares composed config with disk even when inventory still has the promoted hash', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); await runDriftDeployment({ servers: [server] }); const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg'); await writeFile(path, 'changed outside Electric Cherry'); expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]); await runDriftDeployment({ servers: [server] }); expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater')); }); it.each(['running', 'ready-promote'])('defers all drift for a server with a %s beta', async (state) => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n')); const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false }); db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId); const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg'); const betaBytes = await readFile(path); const stray = join(fixture.t5, server, 'configs', 'stray.cfg'); await writeFile(stray, 'also deferred'); expect(await scanServerDrift(server)).toEqual([]); expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 }); expect(await readFile(path)).toEqual(betaBytes); expect(await readFile(stray, 'utf8')).toBe('also deferred'); db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId); expect((await scanServerDrift(server)).map(entry => entry.relPath)).toEqual(['dedicated_sp.cfg', 'stray.cfg']); }); it('reports an unavailable registered directory without hiding drift on another game root', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); const other = 't6_zombie_tranzit_1'; await mkdir(join(fixture.t6, other, 'configs'), { recursive: true }); await writeFile(join(fixture.t6, other, 'configs', 'stray.cfg'), 'untracked'); await seedServerRegistry(); await rm(fixture.t5, { recursive: true }); await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable'); await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable'); expect(existsSync(fixture.t5)).toBe(false); expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]); expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]); }); it('replaces a seeded config once instead of planning both removal and installation', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg'); await writeFile(path, 'seeded config'); expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind: 'config', driftKind: 'replace' })]); expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 }); expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater')); expect(await scanServerDrift(server)).toEqual([]); }); it.each(['identical', 'different'])('reconciles an existing %s scoped file in one pass with an update audit', async (mode) => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const content = Buffer.from('main(){}\n'); const before = mode === 'identical' ? content : Buffer.from('old bytes'); const { script, version } = createScript({ name: 'spawn fix', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'zm_spawn_fix.gsc', content }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const target = join(destOfKind(server, 'script'), 'zm_spawn_fix.gsc'); await writeFile(target, before); expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'zm_spawn_fix.gsc', driftKind: 'replace' })]); expect(await runDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null }); expect(await readFile(target)).toEqual(before); expect(getServerScript(server, 'zm_spawn_fix.gsc')).toBeUndefined(); const result = await runDriftDeployment({ servers: [server] }); expect(result).toMatchObject({ totalRemoved: 0, totalRestored: 1 }); expect(await readFile(target)).toEqual(content); expect(getServerScript(server, 'zm_spawn_fix.gsc')?.script_version_id).toBe(version.id); expect(db().prepare('SELECT verb, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId)) .toEqual([{ verb: 'updated', content_before: before, content_after: content }]); expect(await scanServerDrift(server)).toEqual([]); expect((await runDriftDeployment({ servers: [server] })).deploymentId).toBeNull(); }); it('detects untracked files across script, config, and plugin destinations', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); await seedServerRegistry(); // Drop one stray file in each managed destination. const server = 't5_zombie_theater_1'; await writeFile( join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'main(){}\n' ); await writeFile( join(fixture.t5, server, 'configs', 'stray.cfg'), 'set foo "bar"\n' ); await writeFile( join(fixture.t5, server, 'runtime/plutonium/plugins', 'stray.dll'), Buffer.from([0xde, 0xad]) ); const drift = await scanServerDrift(server); const untracked = drift.filter((d) => d.driftKind === 'untracked'); const byKind = Object.fromEntries(untracked.map((d) => [d.kind, d.relPath])); expect(byKind.script).toBe('stray.gsc'); expect(byKind.config).toBe('stray.cfg'); expect(byKind.plugin).toBe('stray.dll'); expect(untracked.length).toBe(3); }); }); describe('drift rebuild campaigns', () => { it('captures intended changes without writing and follows the linked rebuild without another restart', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { prepareDriftCampaign, createDriftRebuildCampaign } = await import('../../src/lib/server/fleet/drift.js'); const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); await seedServerRegistry(); const server = 't5_zombie_theater_1'; const { script, version } = createScript({ name: 'managed', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'managed.gsc', content: Buffer.from('promoted content') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const target = join(destOfKind(server, 'script'), 'managed.gsc'); await writeFile(target, 'old content'); const stray = join(destOfKind(server, 'config'), 'stray.cfg'); await writeFile(stray, 'untracked'); const prepared = await prepareDriftCampaign(server); const operationId = beginDeployment(server, 'rebuild', 'watchdog'); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(operationId); const campaignId = createDriftRebuildCampaign(server, operationId, 42, prepared); expect(campaignId).toBeTruthy(); expect(await readFile(target, 'utf8')).toBe('old content'); expect(await readFile(stray, 'utf8')).toBe('untracked'); expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId)) .toEqual([ { rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: version.content }, { rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null } ]); await sweep(); expect(db().prepare('SELECT action FROM server_deployments').all()).toEqual([{ action: 'rebuild' }]); expect(db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId)) .toEqual({ provisioning_id: operationId, restart_state: 'pending' }); db().prepare("UPDATE server_deployments SET state = 'done', submitted_at = ?, finished_at = ? WHERE id = ?") .run(new Date().toISOString(), new Date().toISOString(), operationId); await sweep(); expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId)).toEqual({ restart_state: 'done' }); expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaignId)).toMatchObject({ finished_at: expect.any(String) }); }); }); // Deploys a promoted artifact to a scope-matching server, then deletes it // off disk so drift's "missing" pass has something to find. async function promoteAndOrphan( createScript: (i: { name: string; kind: 'script' | 'plugin' | 'baseline' | 'config'; scopeType?: 'game' | 'map'; scopeValue?: string; targetRelPath?: string; content: Buffer; }) => { script: { id: number }; version: { id: number } }, getLatestVersion: (id: number) => { id: number } | undefined, openBeta: (i: { scripts: Array<{ scriptId: number; scriptVersionId: number }>; servers: string[]; waitForHealth: boolean; waitForMatch: boolean; }) => Promise<{ betaId: number }>, promoteBeta: (id: number) => Promise, db: () => { prepare: (q: string) => { run: (...args: unknown[]) => unknown } } ): Promise<{ scriptId: number }> { const created = createScript({ name: 't5 hud', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'hud.gsc', content: Buffer.from('main(){}\n') }); const ver = getLatestVersion(created.script.id)!; const { betaId } = await openBeta({ scripts: [{ scriptId: created.script.id, scriptVersionId: ver.id }], servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false }); db().prepare( `UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?` ).run(new Date().toISOString(), betaId); db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId); await promoteBeta(betaId); return { scriptId: created.script.id }; } describe('missing artifact detection', () => { it('flags a promoted script that is absent on a scope-matching server as missing, then restores it on reconcile', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript, getLatestVersion } = await import( '../../src/lib/server/fleet/scripts.js' ); const { openBeta, promoteBeta } = await import( '../../src/lib/server/fleet/beta.js' ); const { scanServerDrift, runDriftDeployment } = await import( '../../src/lib/server/fleet/drift.js' ); const { db } = await import('../../src/lib/server/db/index.js'); await seedServerRegistry(); await promoteAndOrphan( createScript as never, getLatestVersion as never, openBeta as never, promoteBeta as never, db as never ); const targetPath = join( fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/storage/t5/raw/scripts/sp', 'hud.gsc' ); // Sanity check: promoted script lands on disk. expect(existsSync(targetPath)).toBe(true); // Simulate someone deleting the file out-of-band. await rm(targetPath, { force: true }); const drift = await scanServerDrift('t5_zombie_theater_1'); const missing = drift.filter((d) => d.driftKind === 'missing' && d.kind === 'script'); expect(missing.length).toBe(1); expect(missing[0].relPath).toBe('hud.gsc'); const result = await runDriftDeployment({ servers: ['t5_zombie_theater_1'], dryRun: false, triggeredBy: 'ui' }); expect(result.totalRestored).toBeGreaterThanOrEqual(1); expect(existsSync(targetPath)).toBe(true); const restored = await readFile(targetPath); expect(restored.toString()).toBe('main(){}\n'); }); it('flags a scope-matching server whose composed dedicated_sp.cfg is missing', async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { createScript, getLatestVersion } = await import( '../../src/lib/server/fleet/scripts.js' ); const { openBeta, promoteBeta } = await import( '../../src/lib/server/fleet/beta.js' ); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); const { db } = await import('../../src/lib/server/db/index.js'); await seedServerRegistry(); const baseline = createScript({ name: 't5 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set g_inactivity "190"\n') }); const bVer = getLatestVersion(baseline.script.id)!; const { betaId } = await openBeta({ scripts: [{ scriptId: baseline.script.id, scriptVersionId: bVer.id }], servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false }); db().prepare( `UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?` ).run(new Date().toISOString(), betaId); db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId); await promoteBeta(betaId); const cfgPath = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg'); expect(existsSync(cfgPath)).toBe(true); await rm(cfgPath, { force: true }); const drift = await scanServerDrift('t5_zombie_theater_1'); const missingCfg = drift.filter((d) => d.driftKind === 'missing' && d.kind === 'config'); expect(missingCfg.length).toBe(1); expect(missingCfg[0].relPath).toBe('dedicated_sp.cfg'); }); });