import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { existsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('node:fs', async (importOriginal) => { const original = await importOriginal(); return { ...original, writeFileSync: vi.fn(original.writeFileSync) }; }); vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); let fixture: Fixture; const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2']; const relPath = 'staged.gsc'; const before = Buffer.from('main(){ /* current live script */ }'); const after = Buffer.from('main(){ /* reviewed replacement */ }'); type Source = 'campaign' | 'beta'; const destination = (server = servers[0]) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath); beforeEach(async () => { vi.resetModules(); vi.resetAllMocks(); const actualFs = await vi.importActual('node:fs'); vi.mocked(writeFileSync).mockImplementation(actualFs.writeFileSync); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-27T15:00:00Z')); fixture = await newFixture(servers); vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(komodoRestart).mockResolvedValue({ ok: true }); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.useRealTimers(); }); async function queueChange(source: Source, targets = [servers[0]]) { const { db } = await import('../../src/lib/server/db/index.js'); const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const original = createScript({ name: 'Staged script', kind: 'script', targetRelPath: relPath, scopeType: 'game', scopeValue: 't5', content: before }); const replacement = uploadNewVersion(original.script.id, after); for (const server of targets) { await writeFile(destination(server), before); recordScript(server, relPath, original.version.id, original.version.sha256, before.length, 'fixture'); } const inventory = getServerScript(targets[0], relPath); let sourceId: string; if (source === 'campaign') { db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(replacement.id, original.script.id); const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const result = await runDriftDeployment({ servers: targets }); expect(result.deploymentId).toBeTruthy(); sourceId = result.deploymentId!; } else { const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); const result = await openBeta({ scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }], servers: targets, waitForHealth: false, waitForMatch: false }); sourceId = String(result.betaId); } return { sourceId, inventory, replacement }; } async function tick(source: Source) { const { sweep, betaTick } = await import('../helpers/provisioning.js'); await (source === 'campaign' ? sweep() : betaTick()); } async function expectLiveUnchanged(inventory: Awaited>['inventory'], server = servers[0]) { const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(await readFile(destination(server))).toEqual(before); expect(getServerScript(server, relPath)).toEqual(inventory); } const blockers = ['occupied', 'unknown occupancy', 'missing Komodo', 'missing RCON', 'previous restart', 'rollout capacity'] as const; describe.each(['campaign', 'beta'])('%s staged file activation', (source) => { it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => { const { db } = await import('../../src/lib/server/db/index.js'); const { config } = await import('../../src/lib/server/config.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3); if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null); if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' }); if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' }); if (blocker === 'previous restart' || blocker === 'rollout capacity') { db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)') .run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString()); } const { inventory, replacement } = await queueChange(source); await expectLiveUnchanged(inventory); await tick(source); await expectLiveUnchanged(inventory); expect(komodoRestart).not.toHaveBeenCalled(); // The replacement and its inventory must already be installed when // Komodo sees the activation request, after every admission gate passes. Object.assign(config.komodo, { address: 'http://komodo.test' }); Object.assign(config.rcon, { password: 'test' }); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); db().prepare('DELETE FROM restart_rollout').run(); let bytesAtRestart: Buffer | undefined; let inventoryAtRestart: typeof inventory; vi.mocked(komodoRestart).mockImplementationOnce(async () => { const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); bytesAtRestart = await readFile(destination()); inventoryAtRestart = getServerScript(servers[0], relPath); return { ok: true }; }); await tick(source); expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); expect(bytesAtRestart).toEqual(after); expect(inventoryAtRestart).toMatchObject({ script_version_id: replacement.id, sha256: replacement.sha256, size: after.length }); }); it('does not write when its provisioning request is cancelled during the occupancy probe', async () => { const { inventory } = await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'") .get(servers[0]) as { id: string }; cancelProvisioning(operation.id); return 0; }); await tick(source); await expectLiveUnchanged(inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0])) .toEqual({ state: 'cancelled' }); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); }); it('never applies old staged work to a replacement server created during its occupancy probe', async () => { await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const replacementBytes = Buffer.from('new server instance'); let replacementInventory: ReturnType; vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); await seedServerRegistry(); await writeFile(destination(), replacementBytes); recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement'); replacementInventory = getServerScript(servers[0], relPath); return 0; }); await tick(source); expect(await readFile(destination())).toEqual(replacementBytes); expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0])) .toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) }); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); }); it('preserves existing file permissions when activating staged bytes', async () => { await queueChange(source); await chmod(destination(), 0o640); await tick(source); expect(await readFile(destination())).toEqual(after); expect((await stat(destination())).mode & 0o777).toBe(0o640); }); it('applies the exact queued bytes even if artifact storage changes while waiting', async () => { const { replacement } = await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('UPDATE script_versions SET content = ? WHERE id = ?') .run(Buffer.from('changed after review'), replacement.id); await tick(source); expect(await readFile(destination())).toEqual(after); }); it('keeps the queued payload across an application restart', async () => { const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconGetPlayerCount).mockResolvedValue(2); const { inventory } = await queueChange(source); await tick(source); await expectLiveUnchanged(inventory); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); vi.resetModules(); await tick(source); await expectLiveUnchanged(inventory); const freshRcon = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0); await tick(source); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); expect(await readFile(destination())).toEqual(after); }); it('does not restart or retain rollout capacity when installing staged files fails', async () => { const { inventory } = await queueChange(source); const { db } = await import('../../src/lib/server/db/index.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); // A directory appearing at the destination causes a deterministic I/O // failure even when the test process can bypass ordinary file modes. await rm(destination()); await mkdir(destination()); await tick(source); expect(komodoRestart).not.toHaveBeenCalled(); expect(getServerScript(servers[0], relPath)).toEqual(inventory); expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]); expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0])) .toMatchObject({ state: 'failed', error: expect.any(String) }); }); }); describe('staged source eligibility', () => { it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => { const { sourceId, inventory } = await queueChange('campaign'); const { db } = await import('../../src/lib/server/db/index.js'); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); await sweep(); db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId); await processProvisioning(); await expectLiveUnchanged(inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); }); it('does not install a beta withdrawn after it requests provisioning', async () => { const { sourceId, inventory } = await queueChange('beta'); const { db } = await import('../../src/lib/server/db/index.js'); const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); await betaTick(); db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId); await processProvisioning(); await expectLiveUnchanged(inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); }); }); describe('staged drift removals and review guards', () => { it('defers removing a file and its inventory until the empty-server restart', async () => { const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); await writeFile(destination(), before); recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture'); const inventory = getServerScript(servers[0], relPath); vi.mocked(rconGetPlayerCount).mockResolvedValue(1); await runDriftDeployment({ servers: [servers[0]] }); await expectLiveUnchanged(inventory); await tick('campaign'); await expectLiveUnchanged(inventory); expect(komodoRestart).not.toHaveBeenCalled(); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); let fileExistedAtRestart: boolean | undefined; let inventoryAtRestart: typeof inventory; vi.mocked(komodoRestart).mockImplementationOnce(async () => { fileExistedAtRestart = existsSync(destination()); inventoryAtRestart = getServerScript(servers[0], relPath); return { ok: true }; }); await tick('campaign'); expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); expect(fileExistedAtRestart).toBe(false); expect(inventoryAtRestart).toBeUndefined(); }); it('fails without overwriting a reviewed target changed while its restart was queued', async () => { const { inventory } = await queueChange('campaign'); const externalChange = Buffer.from('changed outside Electric Cherry while queued'); await writeFile(destination(), externalChange); await tick('campaign'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect(await readFile(destination())).toEqual(externalChange); expect(getServerScript(servers[0], relPath)).toEqual(inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0])) .toMatchObject({ error: expect.stringContaining('changed while queued') }); }); }); describe('shared activation and partial writes', () => { it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => { const first = await queueChange('campaign'); const { db } = await import('../../src/lib/server/db/index.js'); const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const second = await runDriftDeployment({ servers: [servers[0]] }); await sweep(); const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[]; expect(linked).toHaveLength(2); expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1); db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId); await processProvisioning(); await expectLiveUnchanged(first.inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' }); db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId); await tick('campaign'); expect(await readFile(destination())).toEqual(after); expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all()) .toEqual([{ state: 'applied' }, { state: 'applied' }]); }); it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => { const campaign = await queueChange('campaign'); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); await sweep(); const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes')); await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }], servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' }); await expectLiveUnchanged(campaign.inventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM betas').all()).toEqual([]); expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId)) .toEqual({ restart_state: 'pending' }); expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId)) .toEqual({ state: 'pending' }); }); it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => { const beta = await queueChange('beta'); const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); // Recreate overlapping work retained from before the admission guard. // The beta was already open; this older drift still expects its original // live bytes and must fail independently of the later restoration. db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)") .run(new Date(Date.now() - 60_000).toISOString()); db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]); queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script', relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift', expectedBeforeSha: sha256Buffer(before) }); await sweep(); await tick('beta'); expect(await readFile(destination())).toEqual(after); const rollback = await rollbackBeta(Number(beta.sourceId)); vi.setSystemTime(Date.now() + 15_000); await tick('campaign'); expect(await readFile(destination())).toEqual(before); expect(komodoRestart).toHaveBeenCalledTimes(2); const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?') .get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string }; const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?') .get(rollback.deploymentId) as { provisioning_id: string }; expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('changed while queued') }); expect(restoration.provisioning_id).not.toBe(old.provisioning_id); }); it('records partial installation without restarting and never automatically retries the failed write', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after }); const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after }); const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc'); for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) { await writeFile(path, before); recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture'); } const secondInventory = getServerScript(servers[0], 'second.gsc'); await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })), servers: [servers[0]], waitForHealth: false, waitForMatch: false }); const actualFs = await vi.importActual('node:fs'); vi.mocked(writeFileSync).mockImplementation((path, content, options) => { if (path === secondPath) throw new Error('Simulated disk write failure'); actualFs.writeFileSync(path, content, options); }); await tick('beta'); expect(await readFile(destination())).toEqual(after); expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id); expect(await readFile(secondPath)).toEqual(before); expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory); expect(komodoRestart).not.toHaveBeenCalled(); expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all()) .toEqual([{ state: 'applied' }, { state: 'applying' }]); expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get()) .toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') }); expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); const attempts = vi.mocked(writeFileSync).mock.calls.length; await tick('beta'); expect(writeFileSync).toHaveBeenCalledTimes(attempts); expect(komodoRestart).not.toHaveBeenCalled(); }); });