feat: unify recovery, updates, and schedules under Maintenance

2026-09-27 16:00:18 -04:00
parent 8ea696417f
commit fc46cb9da4
42 changed files with 1256 additions and 833 deletions
@@ -395,3 +395,36 @@ describe('Watchdog drift campaigns', () => {
        expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
    });
 });
+
+describe('Maintenance recovery policy boundaries', () => {
+    it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => {
+        const policies = await import('../../src/lib/server/maintenance/policies.js');
+        policies.setMaintenancePolicy('health', false);
+        down();
+        for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
+        expect(operations()).toEqual([]);
+        expect(onlyIncident().kind).toBe('health');
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift');
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]);
+        policies.setMaintenancePolicy('drift', false);
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        expect(operations()[0].state).toBe('waiting');
+        expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
+    });
+    it('global pause blocks escalation without disabling the saved health policy', async () => {
+        await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        const policies = await import('../../src/lib/server/maintenance/policies.js');
+        policies.pauseMaintenance(true);
+        advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toHaveLength(1);
+        expect(policies.maintenancePolicies().health).toBe(true);
+        policies.pauseMaintenance(false); await watchdog.watchdogTick(true);
+        expect(operations()).toHaveLength(2);
+        expect(operations()[1].action).toBe('rebuild');
+    });
+});