josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unify recovery, updates, and schedules under Maintenance

Browse Source
This commit is contained in:
josh
2026-09-27 16:00:18 -04:00
parent 8ea696417f
commit fc46cb9da4
42 changed files with 1256 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+114
View File
@@ -233,3 +233,117 @@ describe('persistent maintenance', () => {
expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
});
});
describe('Maintenance policies and runtime updates', () => {
async function outdated() {
const rcon = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
const m = await api(); await m.checkPlutonium(); return m;
}
it('preserves the legacy recovery preference without enabling updates', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run();
const p = await import('../../src/lib/server/maintenance/policies.js');
expect(p.maintenancePolicies()).toEqual({ paused: false, health: false, drift: false, updates: false });
p.setMaintenancePolicy('health', true); p.pauseMaintenance(true); closeDb();
expect(p.maintenancePolicies()).toEqual({ paused: true, health: true, drift: false, updates: false });
expect(p.maintenanceAllowed('health')).toBe(false);
});
it('only automatically updates verified outdated servers and requires opt-in', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?').run(servers[1]);
await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([]);
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
await m.maintenanceTick(); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]);
expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown');
});
it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true);
await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1);
p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000);
await processProvisioning(); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' });
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => {
const m = await outdated();
const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]);
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const failed = m.plutoniumStatus().jobs[0];
expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' });
closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(1);
expect(m.retryMaintenance(failed.id)).toBe(1);
expect(m.plutoniumStatus().jobs).toHaveLength(2);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' });
expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined();
});
it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => {
const m = await outdated(); m.queueUpdates([servers[0]]);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET desired_state = 'stopped'").run();
await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).not.toHaveBeenCalled();
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
expect(m.queueUpdates()).toBe(0);
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true);
});
it('separates scheduled restart success from revision verification and coalesces paused runs', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true);
vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(0);
p.pauseMaintenance(false); await maintenanceTick();
expect(m.plutoniumStatus().jobs).toHaveLength(2);
expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done');
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
});
it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
const m = await outdated();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)').run(servers[0], new Date().toISOString(), new Date().toISOString());
expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
vi.setSystemTime(Date.now() + 3_600_000);
expect(() => m.queueUpdates()).toThrow('published revision');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockClear();
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
await m.checkPlutonium();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true);
});
it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => {
const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
m.queueMaintenance([servers[0]]); await maintenanceTick();
const before = m.plutoniumStatus();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb();
const after = m.plutoniumStatus();
expect(after.schedule).toEqual(before.schedule);
expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' });
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoRestart).toHaveBeenCalledTimes(1);
});
});