Keep held provisioning recovery current (Refs #86)

docs/development/ui-patterns.md

@@ -100,7 +100,8 @@ when updates are unavailable. These clocks are not live regions.
 
 Provisioning detail pages show the requester, absolute start time and duration
 in the summary. Keep the finish time, raw IDs and copy controls in collapsed
-Technical details. Show freshness and poll only while the operation is active.
+Technical details. Show freshness only while the operation is active. Poll active
+operations and finished operations whose recovery capacity is still held.
 Use `OperationTime` for steps, checks and activity: offsets from
 the operation start (`+m:ss` or `+h:mm:ss`), with negative offsets for earlier
 evidence. Missing timestamps stay **Not recorded**.

src/lib/ui/provisioning-presentation.ts

@@ -15,6 +15,12 @@ export function provisioningRequester(operation: Pick<ProvisioningDeployment, 'r
 }
 
 export const activeProvisioningStates = ['queued', 'waiting', 'running', 'awaiting-health'];
+
+/** A finished restart can still release its recovery slot or hand it to a rebuild. */
+export function provisioningNeedsRefresh(detail: ProvisioningDetail): boolean {
+    return activeProvisioningStates.includes(detail.deployment.state) || detail.progress?.recovery?.state === 'held';
+}
+
 export type ProgressState = 'done' | 'current' | 'waiting' | 'failed' | 'pending' | 'cancelled' | 'unknown';
 export interface ProgressStep {
     key: ProvisioningPhase; label: string; state: ProgressState; at: string | null;

src/routes/provisioning/[id]/+page.svelte

@@ -9,7 +9,7 @@
     import { serverTitle } from '$ui/server-labels.js';
     import { createProvisioningRefresh, type ProvisioningRefreshState } from '$ui/provisioning-refresh.js';
     import { confirmAction } from '$ui/confirm.js';
-    import { elapsedTime, isProvisioningDetail, provisioningOutcome, provisioningRequester } from '$ui/provisioning-presentation.js';
+    import { elapsedTime, isProvisioningDetail, provisioningNeedsRefresh, provisioningOutcome, provisioningRequester } from '$ui/provisioning-presentation.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
     import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
@@ -44,6 +44,7 @@
     const canOverride = $derived(data.user?.role === 'admin' && detail?.canOverrideSafety);
     const outcome = $derived(detail ? provisioningOutcome(detail) : null);
     const active = $derived(outcome?.active ?? false);
+    const needsRefresh = $derived(detail ? provisioningNeedsRefresh(detail) : false);
     const duration = $derived(finishedAt || active ? elapsedTime(startedAt, finishedAt ? Date.parse(finishedAt) : now) : 'Not recorded');
     const serverHref = $derived(deployment && deployment.server_exists && !deployment.server_deleted_at ? `/servers/${encodeURIComponent(deployment.server)}` : null);
     const recipeStatus = $derived(detail?.progress?.recipeStatus ?? null);
@@ -58,7 +59,7 @@
         });
     });
     $effect(() => {
-        if (!active) return;
+        if (!needsRefresh) return;
         return pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false);
     });
     onDestroy(() => controller.dispose());
@@ -149,7 +150,6 @@
             {#if detail.progress?.recovery}<div class="recovery-note"><strong>{detail.progress.recovery.state === 'held' ? 'Recovery remains in progress' : detail.progress.recovery.state === 'released' ? 'Recovery capacity released' : detail.progress.recovery.state === 'transferred' ? 'Recovery transferred' : 'Recovery state not confirmed'}</strong><p>{detail.progress.recovery.message}</p><span>Recorded <OperationTime start={deployment.started_at} value={detail.progress.recovery.at} missing="Not recorded" /></span>{#if detail.progress.recovery.operationId && detail.progress.recovery.operationId !== deployment.id}<a href={`/provisioning/${encodeURIComponent(detail.progress.recovery.operationId)}`}>View recovery operation →</a>{/if}</div>{/if}
             {#if serverHref && !['done', 'cancelled'].includes(deployment.state)}<div class="diagnostics"><a href={`${serverHref}?tab=logs`}>Open server logs</a><a href={`${serverHref}?tab=diagnostics`}>Diagnose connection</a>{#if detail.progress?.phase === 'verify-map' || detail.progress?.checks.some(check => check.key === 'map' && check.result !== 'pass')}<a href={`${serverHref}?tab=config`}>Review map configuration</a>{/if}</div>{/if}
             <div class="timing">
-                <div><span>{outcome.active ? 'Total elapsed' : 'Recorded duration'}</span><strong>{duration}</strong></div>
                 {#if outcome.active}<div><span>Time in phase</span><strong>{elapsedTime(timestampISO(detail.progress?.phaseStartedAt), now)}</strong></div>{/if}
                 <div><span>Last evaluated</span><OperationTime start={deployment.started_at} value={detail.progress?.observedAt} missing="Not recorded" /></div>
             </div>

tests/unit/ui-provisioning-detail.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest';
+import { render } from 'svelte/server';
+import { writable } from 'svelte/store';
+import type { ProvisioningDetail, RecoveryObservation } from '../../src/lib/provisioning-detail.js';
+import { provisioningNeedsRefresh } from '../../src/lib/ui/provisioning-presentation.js';
+import ProvisioningPage from '../../src/routes/provisioning/[id]/+page.svelte';
+
+vi.mock('$app/stores', () => ({ page: writable({
+    url: new URL('http://localhost/provisioning/operation-a'), data: { user: null }
+}) }));
+vi.mock('$app/navigation', () => ({ goto: vi.fn() }));
+
+const start = '2026-10-09T12:00:00Z';
+const finish = '2026-10-09T12:01:00Z';
+function fixture(state = 'done', recovery: RecoveryObservation | null = null): ProvisioningDetail {
+    return {
+        deployment: { id: 'operation-a', server: 't5_zombie_theater_1', action: 'restart', state,
+            stage: 'Completed', server_deleted_at: null, server_exists: 1, started_at: start, finished_at: finish,
+            triggered_by: 'watchdog:incident-a', requested_by: 'Maxis', error: null, warnings: '[]' },
+        generatedAt: finish, events: [], activity: { total: 0, latestId: null, failureExcerpt: null },
+        sources: [{ kind: 'watchdog', source_id: 'incident-a', label: 'Deleted watchdog incident', href: null }],
+        recipe: null, blueprintName: null, canResumeCleanup: false, canResumeRebuild: false,
+        canResumeVerification: false, canCancelRemoval: false,
+        collection: { id: 'collection-a', name: 'Evening lobbies' },
+        progress: { version: 1, phase: 'verify-game', phaseStartedAt: '2026-10-09T12:00:50Z', observedAt: finish,
+            blocker: null, checks: [], checkpoints: [], command: null, recovery, recipeStatus: null },
+        agentOperation: { id: 'command-a', hostId: 'host-a', state: 'completed', createdAt: start, updatedAt: finish, error: null },
+        capacity: { evaluatedAt: finish, pools: [] }, warningMessages: []
+    };
+}
+function pageBody(detail: ProvisioningDetail) {
+    return render(ProvisioningPage, { props: { data: { detail, user: null, setupRequired: false } } }).body;
+}
+function recovery(state: RecoveryObservation['state']): RecoveryObservation {
+    return { state, at: finish, message: 'Recovery status updated', ...(state === 'transferred' ? { operationId: 'rebuild-a' } : {}) };
+}
+
+describe('provisioning detail summary and technical details', () => {
+    it('puts requester, absolute start and one duration above the outcome', () => {
+        const body = pageBody(fixture());
+        const summary = body.match(/<p\b[^>]*class="operation-summary[^>]*>[\s\S]*?<\/p>/)?.[0];
+        expect(summary).toBeDefined();
+        expect(summary).toContain('Requested by Maxis');
+        expect(summary).toContain('datetime="2026-10-09T12:00:00.000Z"');
+        expect(summary).toContain('Oct 9, 2026');
+        expect(summary).toContain('took 1m 0s');
+        expect(body.indexOf(summary!)).toBeLessThan(body.indexOf('id="outcome-title"'));
+        expect(body.match(/1m 0s/g)).toHaveLength(1);
+        expect(body).not.toContain('Recorded duration');
+        expect(body).not.toContain('Operation outcome');
+        expect(body).not.toContain('Updates every 5 seconds');
+    });
+
+    it('keeps operation, command and source IDs and copy controls in closed Technical details', () => {
+        const body = pageBody(fixture());
+        const technical = body.match(/<details\b[^>]*class="technical-details[^>]*>[\s\S]*?<\/details>/)?.[0];
+        expect(technical).toBeDefined();
+        expect(technical).toContain('Technical details</summary>');
+        expect(technical!.split('>')[0]).not.toMatch(/\bopen\b/);
+        const outside = body.replace(technical!, '');
+        for (const [value, label] of [
+            ['operation-a', 'operation ID'], ['command-a', 'agent command ID'],
+            ['collection-a', 'collection launch ID'], ['incident-a', 'Deleted watchdog incident source ID']
+        ]) {
+            const code = new RegExp(`<code\\b[^>]*>${value}</code>`);
+            expect(technical).toMatch(code);
+            expect(outside).not.toMatch(code);
+            expect(technical).toContain(`aria-label="Copy ${label}"`);
+            expect(outside).not.toContain(`aria-label="Copy ${label}"`);
+        }
+        expect(technical).toContain('Copy link');
+        expect(outside).not.toContain('Copy link');
+        expect(technical).toContain('datetime="2026-10-09T12:01:00.000Z"');
+        expect(outside).toContain('href="/collections/launches/collection-a"');
+        expect(outside).toContain('Source record no longer available');
+    });
+
+    it('shows one elapsed duration for active work and preserves phase timing and freshness', () => {
+        const detail = fixture('running');
+        detail.deployment.finished_at = null;
+        const body = pageBody(detail);
+        expect(body).toContain('1m 0s elapsed');
+        expect(body.match(/1m 0s/g)).toHaveLength(1);
+        expect(body).not.toContain('Total elapsed');
+        expect(body).toContain('Time in phase');
+        expect(body).toContain('10s');
+        expect(body).toContain('Last evaluated');
+        expect(body).toContain('Updates every 5 seconds');
+    });
+
+    it('does not invent a duration when the finish time is missing', () => {
+        const detail = fixture('interrupted');
+        detail.deployment.finished_at = null;
+        const body = pageBody(detail);
+        expect(body).toContain('Duration: Not recorded');
+        expect(body).not.toContain('took ');
+        expect(body).not.toContain(' elapsed');
+        expect(body).not.toContain('Updates every 5 seconds');
+    });
+});
+
+describe('provisioning refresh policy', () => {
+    it.each(['queued', 'waiting', 'running', 'awaiting-health'])('refreshes %s operations', state => {
+        expect(provisioningNeedsRefresh(fixture(state))).toBe(true);
+    });
+
+    it.each(['done', 'failed', 'interrupted', 'warning', 'cancelled'])('stops refreshing %s operations without held recovery', state => {
+        const detail = fixture(state);
+        expect(provisioningNeedsRefresh(detail)).toBe(false);
+        for (const value of ['released', 'transferred', 'unknown'] as const) {
+            detail.progress!.recovery = recovery(value);
+            expect(provisioningNeedsRefresh(detail)).toBe(false);
+        }
+        detail.progress = null;
+        expect(provisioningNeedsRefresh(detail)).toBe(false);
+    });
+
+    it.each(['released', 'transferred'] as const)('keeps failed restart recovery refreshing until %s without marking it live', state => {
+        const detail = fixture('failed', recovery('held'));
+        expect(provisioningNeedsRefresh(detail)).toBe(true);
+        const held = pageBody(detail);
+        expect(held).toContain('Recovery capacity held');
+        expect(held).not.toContain('Updates every 5 seconds');
+        expect(held).not.toContain('View recovery operation');
+
+        detail.progress!.recovery = recovery(state);
+        expect(provisioningNeedsRefresh(detail)).toBe(false);
+        const settled = pageBody(detail);
+        expect(settled).not.toContain('Recovery capacity held');
+        expect(settled).not.toContain('Updates every 5 seconds');
+        expect(settled).toContain(state === 'released' ? 'Recovery capacity released' : 'Recovery transferred');
+        if (state === 'transferred') {
+            expect(settled).toContain('href="/provisioning/rebuild-a"');
+            expect(settled).toContain('View recovery operation');
+        }
+    });
+});