Suppress maintenance watchers during provisioning

Keep maintenance observations and automatic requests clear of active lifecycle operations and retained recovery capacity while preserving owned recovery and explicit retries. Fixes #55
2026-10-01 11:54:22 -04:00
parent a3cd3ee585
commit 0cb48c435e
8 changed files with 366 additions and 18 deletions
@@ -194,6 +194,50 @@ describe('Watchdog detection and ownership', () => {
        expect(incidents()).toEqual([]);
        expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
    });
+
+    it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])(
+        'suppresses health and drift throughout %s provisioning, then resumes', async state => {
+            const retained = state === 'retained recovery';
+            const now = new Date().toISOString();
+            database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
+                VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`)
+                .run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now);
+            if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at)
+                VALUES (?, 'manual-operation', ?)`).run(server, now);
+            await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
+            down();
+            for (let n = 0; n < 4; n++) {
+                await watchdog.watchdogTick(true);
+                advance(watchdog.WATCHDOG_SCAN_MS);
+            }
+            expect(incidents()).toEqual([]);
+            expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
+            expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+            expect(operations()).toHaveLength(1);
+
+            database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run();
+            if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server);
+            await watchdog.watchdogTick(true);
+            expect(incidents()).toEqual(expect.arrayContaining([
+                expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }),
+                expect.objectContaining({ kind: 'drift', state: 'recovering' })
+            ]));
+            expect(operations()).toHaveLength(2);
+            expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' });
+        }
+    );
+
+    it('stops probing when provisioning takes ownership during the container inspection', async () => {
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
+            expect(requestLifecycle(server, 'restart').ok).toBe(true);
+            return { ok: true, state: 'running' };
+        });
+        await watchdog.watchdogTick(true);
+        expect(incidents()).toEqual([]);
+        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
+    });
 });

 describe('Watchdog recovery persistence', () => {
@@ -254,6 +298,44 @@ describe('Watchdog recovery persistence', () => {
        expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
    });

+    it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => {
+        const row = await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        await watchdog.watchdogTick(true);
+        vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
+        await processProvisioning();
+        await watchdog.watchdogTick(true);
+        const failed = operations()[1];
+        expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' });
+        expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 });
+        const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server);
+        expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null });
+
+        await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
+        vi.mocked(komodo.komodoInspectHealthy).mockClear();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        await watchdog.watchdogTick(true);
+        expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
+        expect(operations()).toHaveLength(2);
+
+        expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true });
+        await watchdog.watchdogTick(true);
+        expect(operations()).toHaveLength(2);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        const retry = operations()[2];
+        expect(retry).toMatchObject({ action: 'restart', state: 'waiting' });
+        expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 });
+        expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null });
+        expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3);
+
+        await processProvisioning();
+        expect(komodo.komodoRestart).toHaveBeenCalledTimes(2);
+        expect(operations()[2].state).toBe('awaiting-health');
+    });
+
    it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
        down();
        await watchdog.watchdogTick(true);