josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Defer server file changes until guarded restart activation

Browse Source 
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
This commit is contained in:
josh
2026-09-27 23:06:16 -04:00
parent eb2a852fdc
commit fb48f64de1
36 changed files with 1634 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/routes/campaigns/[id]/+page.svelte
+28 -2
View File
@@ -20,6 +20,7 @@
id: number; server: string; verb: string; rel_path: string;
sha256_before: string | null; sha256_after: string | null;
size_before: number | null; size_after: number | null;
apply_state: 'pending' | 'applying' | 'applied' | 'cancelled';
}
interface ServerRow {
deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number;
@@ -40,6 +41,8 @@
let detail = $state<Detail | null>(null);
let loadError = $state('');
let resuming = $state(false);
let resumeError = $state('');
let request = 0;
let diffOpen = $state(false);
let diffError = $state('');
@@ -48,7 +51,8 @@
let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null);
let retryFile = $state<FileRow | null>(null);
const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const withdrawn = $derived(detail?.servers.filter(server => ['skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && !['done', 'skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => {
const rank: Record<string, number> = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 };
@@ -73,6 +77,16 @@
onMount(() => pollVisible(loadDetail, 5000, false));
async function resume() {
resuming = true; resumeError = '';
try {
const response = await fetch(`/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, { method: 'POST' });
if (!response.ok) throw new Error((await response.json()).error ?? 'Could not resume campaign.');
await loadDetail();
} catch (error) { resumeError = (error as Error).message; }
finally { resuming = false; }
}
async function loadDiff(f: FileRow) {
retryFile = f;
const id = $page.params.id;
@@ -123,10 +137,21 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome">
<h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2>
<p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}</p>
{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>
{:else if withdrawn}<p>Withdrawn changes will not be activated by this campaign.</p>
{:else}<p>Every recorded server completed activation.</p>{/if}
</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Maintenance · <a href={`/maintenance/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
{#if detail.servers.some(server => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))}
<div><Button onclick={resume} loading={resuming} disabled={resuming}>Resume when empty</Button><p>Apply held changes and restart after the safety checks pass.</p></div>
{/if}
{#if resumeError}<p class="err" role="alert">{resumeError}</p>{/if}
<Card padding="lg">
<div class="meta-grid">
<div class="kv"><span class="k">Started</span><span class="v mono">{fmt(detail.deployment.started_at)}</span></div>
@@ -182,6 +207,7 @@
{#each files as f (f.id)}
<button class="file-row" onclick={() => loadDiff(f)}>
<Badge variant={verbVariant(f.verb)} size="sm" uppercase>{f.verb}</Badge>
{#if f.apply_state !== 'applied'}<Badge variant="neutral" size="sm">{f.apply_state === 'pending' ? 'Queued' : f.apply_state === 'applying' ? 'Needs inspection' : 'Cancelled'}</Badge>{/if}
<span class="path">{f.rel_path}</span>
<span class="sizes">
{#if f.size_before !== null}{f.size_before} B{/if}