464 lines
28 KiB
TypeScript
464 lines
28 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
import { existsSync, writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('node:fs', async (importOriginal) => {
|
|
const original = await importOriginal<typeof import('node:fs')>();
|
|
return { ...original, writeFileSync: vi.fn(original.writeFileSync) };
|
|
});
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
|
|
const relPath = 'staged.gsc';
|
|
const before = Buffer.from('main(){ /* current live script */ }');
|
|
const after = Buffer.from('main(){ /* reviewed replacement */ }');
|
|
type Source = 'campaign' | 'beta';
|
|
const destination = (server = servers[0]) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath);
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.resetAllMocks();
|
|
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
|
|
vi.mocked(writeFileSync).mockImplementation(actualFs.writeFileSync);
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-27T15:00:00Z'));
|
|
fixture = await newFixture(servers);
|
|
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
|
vi.stubEnv('KOMODO_API_KEY', 'test');
|
|
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
|
vi.stubEnv('RCON_PASSWORD', 'test');
|
|
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rconStatusResult).mockResolvedValue({
|
|
ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
async function queueChange(source: Source, targets = [servers[0]]) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const original = createScript({ name: 'Staged script', kind: 'script', targetRelPath: relPath,
|
|
scopeType: 'game', scopeValue: 't5', content: before });
|
|
const replacement = uploadNewVersion(original.script.id, after);
|
|
for (const server of targets) {
|
|
await writeFile(destination(server), before);
|
|
recordScript(server, relPath, original.version.id, original.version.sha256, before.length, 'fixture');
|
|
}
|
|
const inventory = getServerScript(targets[0], relPath);
|
|
let sourceId: string;
|
|
if (source === 'campaign') {
|
|
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(replacement.id, original.script.id);
|
|
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
|
const result = await runDriftDeployment({ servers: targets });
|
|
expect(result.deploymentId).toBeTruthy();
|
|
sourceId = result.deploymentId!;
|
|
} else {
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const result = await openBeta({ scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }],
|
|
servers: targets, waitForHealth: false, waitForMatch: false });
|
|
sourceId = String(result.betaId);
|
|
}
|
|
return { sourceId, inventory, replacement };
|
|
}
|
|
|
|
async function tick(source: Source) {
|
|
const { sweep, betaTick } = await import('../helpers/provisioning.js');
|
|
await (source === 'campaign' ? sweep() : betaTick());
|
|
}
|
|
|
|
async function expectLiveUnchanged(inventory: Awaited<ReturnType<typeof queueChange>>['inventory'], server = servers[0]) {
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(await readFile(destination(server))).toEqual(before);
|
|
expect(getServerScript(server, relPath)).toEqual(inventory);
|
|
}
|
|
|
|
const blockers = ['occupied', 'unknown occupancy', 'missing Komodo', 'missing RCON', 'previous restart', 'rollout capacity'] as const;
|
|
|
|
describe.each<Source>(['campaign', 'beta'])('%s staged file activation', (source) => {
|
|
it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
|
if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
|
if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' });
|
|
if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' });
|
|
if (blocker === 'previous restart' || blocker === 'rollout capacity') {
|
|
db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)')
|
|
.run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString());
|
|
}
|
|
const { inventory, replacement } = await queueChange(source);
|
|
await expectLiveUnchanged(inventory);
|
|
await tick(source);
|
|
await expectLiveUnchanged(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
|
|
// The replacement and its inventory must already be installed when
|
|
// Komodo sees the activation request, after every admission gate passes.
|
|
Object.assign(config.komodo, { address: 'http://komodo.test' });
|
|
Object.assign(config.rcon, { password: 'test' });
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
|
db().prepare('DELETE FROM restart_rollout').run();
|
|
let bytesAtRestart: Buffer | undefined;
|
|
let inventoryAtRestart: typeof inventory;
|
|
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
bytesAtRestart = await readFile(destination());
|
|
inventoryAtRestart = getServerScript(servers[0], relPath);
|
|
return { ok: true };
|
|
});
|
|
await tick(source);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect(bytesAtRestart).toEqual(after);
|
|
expect(inventoryAtRestart).toMatchObject({
|
|
script_version_id: replacement.id, sha256: replacement.sha256, size: after.length
|
|
});
|
|
});
|
|
|
|
it('does not write when its provisioning request is cancelled during the occupancy probe', async () => {
|
|
const { inventory } = await queueChange(source);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
|
const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'")
|
|
.get(servers[0]) as { id: string };
|
|
cancelProvisioning(operation.id);
|
|
return 0;
|
|
});
|
|
await tick(source);
|
|
await expectLiveUnchanged(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0]))
|
|
.toEqual({ state: 'cancelled' });
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
});
|
|
|
|
it('never applies old staged work to a replacement server created during its occupancy probe', async () => {
|
|
await queueChange(source);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const replacementBytes = Buffer.from('new server instance');
|
|
let replacementInventory: ReturnType<typeof getServerScript>;
|
|
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
|
await seedServerRegistry();
|
|
await writeFile(destination(), replacementBytes);
|
|
recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement');
|
|
replacementInventory = getServerScript(servers[0], relPath);
|
|
return 0;
|
|
});
|
|
await tick(source);
|
|
expect(await readFile(destination())).toEqual(replacementBytes);
|
|
expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0]))
|
|
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
});
|
|
|
|
it('preserves existing file permissions when activating staged bytes', async () => {
|
|
await queueChange(source);
|
|
await chmod(destination(), 0o640);
|
|
await tick(source);
|
|
expect(await readFile(destination())).toEqual(after);
|
|
expect((await stat(destination())).mode & 0o777).toBe(0o640);
|
|
});
|
|
|
|
it('applies the exact queued bytes even if artifact storage changes while waiting', async () => {
|
|
const { replacement } = await queueChange(source);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?')
|
|
.run(Buffer.from('changed after review'), replacement.id);
|
|
await tick(source);
|
|
expect(await readFile(destination())).toEqual(after);
|
|
});
|
|
|
|
it('keeps the queued payload across an application restart', async () => {
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(2);
|
|
const { inventory } = await queueChange(source);
|
|
await tick(source);
|
|
await expectLiveUnchanged(inventory);
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
vi.resetModules();
|
|
await tick(source);
|
|
await expectLiveUnchanged(inventory);
|
|
const freshRcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0);
|
|
await tick(source);
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect(await readFile(destination())).toEqual(after);
|
|
});
|
|
|
|
it('does not restart or retain rollout capacity when installing staged files fails', async () => {
|
|
const { inventory } = await queueChange(source);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
// A directory appearing at the destination causes a deterministic I/O
|
|
// failure even when the test process can bypass ordinary file modes.
|
|
await rm(destination());
|
|
await mkdir(destination());
|
|
await tick(source);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
|
|
expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]);
|
|
expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0]))
|
|
.toMatchObject({ state: 'failed', error: expect.any(String) });
|
|
});
|
|
});
|
|
|
|
describe('staged source eligibility', () => {
|
|
it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => {
|
|
const { sourceId, inventory } = await queueChange('campaign');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await sweep();
|
|
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId);
|
|
await processProvisioning();
|
|
await expectLiveUnchanged(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
});
|
|
|
|
it('does not install a beta withdrawn after it requests provisioning', async () => {
|
|
const { sourceId, inventory } = await queueChange('beta');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await betaTick();
|
|
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId);
|
|
await processProvisioning();
|
|
await expectLiveUnchanged(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
});
|
|
});
|
|
|
|
|
|
describe('staged drift removals and review guards', () => {
|
|
it('defers removing a file and its inventory until the empty-server restart', async () => {
|
|
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
|
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await writeFile(destination(), before);
|
|
recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture');
|
|
const inventory = getServerScript(servers[0], relPath);
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(1);
|
|
await runDriftDeployment({ servers: [servers[0]] });
|
|
await expectLiveUnchanged(inventory);
|
|
await tick('campaign');
|
|
await expectLiveUnchanged(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
|
let fileExistedAtRestart: boolean | undefined;
|
|
let inventoryAtRestart: typeof inventory;
|
|
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
|
|
fileExistedAtRestart = existsSync(destination());
|
|
inventoryAtRestart = getServerScript(servers[0], relPath);
|
|
return { ok: true };
|
|
});
|
|
await tick('campaign');
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect(fileExistedAtRestart).toBe(false);
|
|
expect(inventoryAtRestart).toBeUndefined();
|
|
});
|
|
|
|
it('fails without overwriting a reviewed target changed while its restart was queued', async () => {
|
|
const { inventory } = await queueChange('campaign');
|
|
const externalChange = Buffer.from('changed outside Electric Cherry while queued');
|
|
await writeFile(destination(), externalChange);
|
|
await tick('campaign');
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(await readFile(destination())).toEqual(externalChange);
|
|
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0]))
|
|
.toMatchObject({ error: expect.stringContaining('changed while queued') });
|
|
});
|
|
});
|
|
|
|
|
|
describe('shared activation and partial writes', () => {
|
|
it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => {
|
|
const first = await queueChange('campaign');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const second = await runDriftDeployment({ servers: [servers[0]] });
|
|
await sweep();
|
|
const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[];
|
|
expect(linked).toHaveLength(2);
|
|
expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1);
|
|
db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId);
|
|
await processProvisioning();
|
|
await expectLiveUnchanged(first.inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' });
|
|
db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId);
|
|
await tick('campaign');
|
|
expect(await readFile(destination())).toEqual(after);
|
|
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
|
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
|
|
.toEqual([{ state: 'applied' }, { state: 'applied' }]);
|
|
});
|
|
|
|
it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => {
|
|
const campaign = await queueChange('campaign');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
await sweep();
|
|
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes'));
|
|
await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }],
|
|
servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
|
|
await expectLiveUnchanged(campaign.inventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM betas').all()).toEqual([]);
|
|
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId))
|
|
.toEqual({ restart_state: 'pending' });
|
|
expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId))
|
|
.toEqual({ state: 'pending' });
|
|
});
|
|
|
|
it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => {
|
|
const beta = await queueChange('beta');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
// Recreate overlapping work retained from before the admission guard.
|
|
// The beta was already open; this older drift still expects its original
|
|
// live bytes and must fail independently of the later restoration.
|
|
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)")
|
|
.run(new Date(Date.now() - 60_000).toISOString());
|
|
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
|
VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]);
|
|
queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script',
|
|
relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift',
|
|
expectedBeforeSha: sha256Buffer(before) });
|
|
await sweep();
|
|
await tick('beta');
|
|
expect(await readFile(destination())).toEqual(after);
|
|
const rollback = await rollbackBeta(Number(beta.sourceId));
|
|
vi.setSystemTime(Date.now() + 15_000);
|
|
await tick('campaign');
|
|
expect(await readFile(destination())).toEqual(before);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
|
const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?')
|
|
.get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string };
|
|
const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?')
|
|
.get(rollback.deploymentId) as { provisioning_id: string };
|
|
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('Superseded by rollback') });
|
|
expect(restoration.provisioning_id).not.toBe(old.provisioning_id);
|
|
});
|
|
|
|
it('records partial installation without restarting and never automatically retries the failed write', async () => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after });
|
|
const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after });
|
|
const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc');
|
|
for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) {
|
|
await writeFile(path, before);
|
|
recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture');
|
|
}
|
|
const secondInventory = getServerScript(servers[0], 'second.gsc');
|
|
await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })),
|
|
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
|
|
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
|
|
vi.mocked(writeFileSync).mockImplementation((path, content, options) => {
|
|
if (path === secondPath) throw new Error('Simulated disk write failure');
|
|
actualFs.writeFileSync(path, content, options);
|
|
});
|
|
await tick('beta');
|
|
expect(await readFile(destination())).toEqual(after);
|
|
expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id);
|
|
expect(await readFile(secondPath)).toEqual(before);
|
|
expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
|
|
.toEqual([{ state: 'applied' }, { state: 'applying' }]);
|
|
expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get())
|
|
.toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') });
|
|
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
|
|
const attempts = vi.mocked(writeFileSync).mock.calls.length;
|
|
await tick('beta');
|
|
expect(writeFileSync).toHaveBeenCalledTimes(attempts);
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('rollback supersedes provisioning', () => {
|
|
it.each(['when-empty', 'manual'] as const)('replaces an unhealthy beta restart with a %s rollback without waiting for recovery', async restart => {
|
|
const beta = await queueChange('beta');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
|
const { komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await tick('beta');
|
|
const old = db().prepare('SELECT provisioning_id FROM beta_servers WHERE beta_id=?').get(beta.sourceId) as { provisioning_id: string };
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'awaiting-health' });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'container failed to start' });
|
|
const rollback = await rollbackBeta(Number(beta.sourceId), { restart });
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'cancelled' });
|
|
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(old.provisioning_id)).toEqual([]);
|
|
const replacement = db().prepare('SELECT provisioning_id,restart_state FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as { provisioning_id: string; restart_state: string };
|
|
expect(replacement.provisioning_id).not.toBe(old.provisioning_id);
|
|
await tick('campaign');
|
|
expect(await readFile(destination())).toEqual(restart === 'manual' ? after : before);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(restart === 'manual' ? 1 : 2);
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(replacement.provisioning_id))
|
|
.toEqual({ state: restart === 'manual' ? 'waiting' : 'awaiting-health' });
|
|
});
|
|
});
|