josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Watchdog recovery and drift rebuild workflows

Browse Source
This commit is contained in:
josh
2026-09-27 15:13:38 -04:00
parent a7044f21a1
commit b9f5902945
36 changed files with 2131 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/drift-kinds.test.ts
+147
View File
@@ -73,6 +73,112 @@ describe('reviewed drift plans', () => {
});
describe('drift scanning', () => {
it.each([
['script', 'moved'], ['script', 'unset'], ['plugin', 'moved'], ['plugin', 'unset']
] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll';
const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null });
expect(getServerScript(server, relPath)).toBeDefined();
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]);
await runDriftDeployment({ servers: [server] });
expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false);
expect(getServerScript(server, relPath)).toBeUndefined();
});
it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const relPath = kind === 'script' ? 'managed.gsc' : 'managed.dll';
const content = Buffer.from('promoted content');
const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
const inventory = getServerScript(server, relPath);
const path = join(destOfKind(server, kind), relPath);
await writeFile(path, 'changed outside Electric Cherry');
expect(getServerScript(server, relPath)).toEqual(inventory);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(content);
expect(await scanServerDrift(server)).toEqual([]);
});
it('compares composed config with disk even when inventory still has the promoted hash', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'changed outside Electric Cherry');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
});
it.each(['running', 'ready-promote'])('defers all drift for a server with a %s beta', async (state) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n'));
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false });
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
const betaBytes = await readFile(path);
const stray = join(fixture.t5, server, 'configs', 'stray.cfg');
await writeFile(stray, 'also deferred');
expect(await scanServerDrift(server)).toEqual([]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await readFile(path)).toEqual(betaBytes);
expect(await readFile(stray, 'utf8')).toBe('also deferred');
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
expect((await scanServerDrift(server)).map(entry => entry.relPath)).toEqual(['dedicated_sp.cfg', 'stray.cfg']);
});
it('reports an unavailable registered directory without hiding drift on another game root', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const other = 't6_zombie_tranzit_1';
await mkdir(join(fixture.t6, other, 'configs'), { recursive: true });
await writeFile(join(fixture.t6, other, 'configs', 'stray.cfg'), 'untracked');
await seedServerRegistry();
await rm(fixture.t5, { recursive: true });
await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable');
await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
expect(existsSync(fixture.t5)).toBe(false);
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]);
});
it('replaces a seeded config once instead of planning both removal and installation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
@@ -152,6 +258,47 @@ describe('drift scanning', () => {
});
});
describe('drift rebuild campaigns', () => {
it('captures intended changes without writing and follows the linked rebuild without another restart', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { prepareDriftCampaign, createDriftRebuildCampaign } = await import('../../src/lib/server/fleet/drift.js');
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'managed', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'managed.gsc', content: Buffer.from('promoted content') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const target = join(destOfKind(server, 'script'), 'managed.gsc');
await writeFile(target, 'old content');
const stray = join(destOfKind(server, 'config'), 'stray.cfg');
await writeFile(stray, 'untracked');
const prepared = await prepareDriftCampaign(server);
const operationId = beginDeployment(server, 'rebuild', 'watchdog');
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(operationId);
const campaignId = createDriftRebuildCampaign(server, operationId, 42, prepared);
expect(campaignId).toBeTruthy();
expect(await readFile(target, 'utf8')).toBe('old content');
expect(await readFile(stray, 'utf8')).toBe('untracked');
expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId))
.toEqual([
{ rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: version.content },
{ rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null }
]);
await sweep();
expect(db().prepare('SELECT action FROM server_deployments').all()).toEqual([{ action: 'rebuild' }]);
expect(db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId))
.toEqual({ provisioning_id: operationId, restart_state: 'pending' });
db().prepare("UPDATE server_deployments SET state = 'done', submitted_at = ?, finished_at = ? WHERE id = ?")
.run(new Date().toISOString(), new Date().toISOString(), operationId);
await sweep();
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId)).toEqual({ restart_state: 'done' });
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaignId)).toMatchObject({ finished_at: expect.any(String) });
});
});
// Deploys a promoted artifact to a scope-matching server, then deletes it
// off disk so drift's "missing" pass has something to find.
async function promoteAndOrphan(