Defer server file changes until guarded restart activation

Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.

src/routes/api/betas/[id]/+server.ts

@@ -33,7 +33,9 @@ export async function GET({ params }: { params: { id: string } }) {
     const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
     const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined;
     const promotion = campaign ? { ...campaign,
-        pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n,
+        pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled','skipped')").get(campaign.id) as { n: number }).n,
         failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
-    return json({ beta, scripts, servers, events, test: testView(id), promotion });
+    const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1")
+        .get(id) as { id: string } | undefined;
+    return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null });
 }

src/routes/api/betas/[id]/retry/+server.ts

@@ -3,6 +3,8 @@ import { db } from '$lib/server/db/index.js';
 import { emit, type BetaStage } from '$lib/server/pipeline/bus.js';
 import { storedPlan } from '$lib/server/fleet/beta-tests.js';
 import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
+import { deployBetaServers, getBeta, isActiveBetaServer } from '$lib/server/fleet/beta.js';
+import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -14,18 +16,33 @@ function parseId(raw: string): number | null {
 export async function POST({ params, request }: { params: { id: string }; request: Request }) {
     const betaId = parseId(params.id);
     if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
+    const beta = getBeta(betaId);
+    if (!beta || !['running', 'ready-promote'].includes(beta.state)) return json({ ok: false, error: 'Beta is closed or missing' }, { status: 409 });
     if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 });
     const body = (await request.json().catch(() => ({}))) as { server?: string };
     if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
 
     const row = db()
-        .prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
+        .prepare('SELECT stage, deployed_at, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
         .get(betaId, body.server) as
-        | { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
+        | { stage: string; deployed_at: string | null; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
         | undefined;
     if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 });
-    if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
+    if (!isActiveBetaServer(betaId, body.server)) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
     if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 });
+    if (stagedFileChanges('beta', String(betaId), body.server).some(change => change.state === 'applying')) {
+        return json({ ok: false, error: 'A file write was interrupted. Inspect the server and roll back before opening a fresh beta.' }, { status: 409 });
+    }
+
+    // Preparation may have failed before any bytes were queued (for example,
+    // an overlay without a baseline). Retrying must prepare its payload first.
+    if (!row.deployed_at && !row.restarted_at) {
+        try { await deployBetaServers(betaId, [body.server]); }
+        catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 409 }); }
+        const prepared = db().prepare('SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?')
+            .get(betaId, body.server) as { stage: string; error: string | null };
+        if (prepared.stage === 'failed') return json({ ok: false, error: prepared.error }, { status: 409 });
+    }
 
     // Best-effort resume point: if we already got past restart, go back to
     // health check; otherwise back to waiting for the server to be empty.

src/routes/api/betas/[id]/test/+server.ts

@@ -1,7 +1,8 @@
 import { json } from '@sveltejs/kit';
 import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
 import { db } from '$lib/server/db/index.js';
-import { getBeta } from '$lib/server/fleet/beta.js';
+import { deployBetaServers, getBeta } from '$lib/server/fleet/beta.js';
+import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
 import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
 import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
 import { parseBetaPlan } from '$lib/beta-plan.js';
@@ -52,7 +53,16 @@ export async function POST({ params, request }: { params: { id: string }; reques
             // Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
             // operators open another beta if the overall testing window has expired.
             if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
-            if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta');
+            if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
+                throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
+            }
+            const prepare = db().prepare("SELECT server FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL AND restarted_at IS NULL")
+                .all(id) as { server: string }[];
+            if (prepare.length) {
+                await deployBetaServers(id, prepare.map(row => row.server));
+                const failed = db().prepare("SELECT error FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL").get(id) as { error: string | null } | undefined;
+                if (failed) throw new Error(failed.error ?? 'Could not prepare beta files');
+            }
             db().transaction(() => {
                 const servers = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ?').all(id) as { server: string }[];
                 for (const { server } of servers) releaseRestartRequest(server, { kind: 'beta', id: String(id) });

src/routes/api/campaigns/[id]/+server.ts

@@ -17,7 +17,8 @@ export async function GET({ params }: { params: { id: string } }) {
         .prepare(
             `SELECT id, server, verb, rel_path, sha256_before, sha256_after,
                     LENGTH(content_before) AS size_before,
-                    LENGTH(content_after)  AS size_after
+                    LENGTH(content_after)  AS size_after,
+                    COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state
              FROM deployment_files
              WHERE deployment_id = ?
              ORDER BY server, rel_path`

src/routes/api/campaigns/[id]/resume/+server.ts

@@ -0,0 +1,25 @@
+import { json } from '@sveltejs/kit';
+import { db } from '$lib/server/db/index.js';
+import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    const changed = db().transaction(() => {
+        const rows = db().prepare(`SELECT ds.server, d.state FROM deployment_servers ds
+            LEFT JOIN server_deployments d ON d.id = ds.provisioning_id
+            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`)
+            .all(params.id) as { server: string; state: string | null }[];
+        for (const row of rows) {
+            if (row.state && ['failed','interrupted','cancelled'].includes(row.state)) {
+                releaseRestartRequest(row.server, { kind: 'campaign', id: params.id });
+            }
+        }
+        const result = db().prepare(`UPDATE deployment_servers SET restart_state = 'pending', error = NULL
+            WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`).run(params.id);
+        if (result.changes) db().prepare('UPDATE deployments SET finished_at = NULL WHERE id = ?').run(params.id);
+        return result.changes;
+    })();
+    if (!changed) return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 });
+    void kick().catch(err => console.error('[campaign] activation kick failed:', err));
+    return json({ ok: true }, { status: 202 });
+}

src/routes/betas/[id]/+page.svelte

@@ -31,6 +31,7 @@
     let servers = $state<BetaServer[]>([]);
     let events = $state<ActivityEvent[]>([]);
     let promotion = $state<Promotion|null>(null);
+    let rollbackCampaign = $state<{ id: string }|null>(null);
     let loading = $state(true), loadError = $state(''), flash = $state(''), success = $state('');
     let acting = $state<string|null>(null), request = 0;
     let reviewOpen = $state(false), preview = $state<BetaPromotionPreview|null>(null), reviewError = $state('');
@@ -44,7 +45,7 @@
     const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length);
     const ranked = $derived([...servers].sort((a,b) => (a.stage === 'failed' ? 0 : a.stage === 'ready' ? 2 : 1) - (b.stage === 'failed' ? 0 : b.stage === 'ready' ? 2 : 1) || a.server.localeCompare(b.server)));
     const waiting = $derived(ranked.filter(server => server.stage !== 'ready'));
-    const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check activation on additional servers.' : 'Reuse these files and this plan when you are ready to test again.'
+    const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.'
         : hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.'
         : test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.'
         : test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.'
@@ -60,6 +61,7 @@
             const body = await res.json();
             if (current !== request || requestedId !== id) return;
             test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null;
+            rollbackCampaign = body.rollback ?? null;
             now = Date.now(); loadError = '';
         } catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; }
         finally { if (current === request && requestedId === id) loading = false; }
@@ -86,20 +88,20 @@
                 throw new Error(body.error ?? 'Promotion could not be started.');
             }
             reviewOpen = false;
-            success = `Promotion created. ${body.result.awaitingActivation ?? 0} additional servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`;
+            success = `Promotion created. ${body.result.awaitingActivation ?? 0} servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`;
             await load();
         } catch (error) { reviewError = (error as Error).message; }
         finally { acting = null; }
     }
     async function rollback() {
         const reject = beta?.state === 'ready-promote';
-        const restarts = test?.plan.rollbackRestart === 'manual' ? 'Restart servers manually to load the restored files.' : 'Restarts wait for empty servers.';
+        const restarts = test?.plan.rollbackRestart === 'manual' ? 'Resume the rollback campaign when ready; files stay unchanged until safety checks pass.' : 'Files are restored just before restarting, after safety checks pass.';
         if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.filter(server => !server.server_deleted_at).length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return;
         acting = 'rollback'; flash = ''; success = '';
         try {
             const response = await fetch(`/api/betas/${id}/rollback`,{method:'POST'}), body = await response.json();
             if (!response.ok || !body.ok) throw new Error(body.error ?? 'Rollback failed.');
-            success = `Original files restored. ${restarts}`; await load();
+            success = `Rollback queued. ${restarts}`; await load();
         } catch(error) { flash = (error as Error).message; }
         finally { acting = null; }
     }
@@ -142,6 +144,7 @@
             {#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
             {#if closed}<Button href={`/betas/new?reuse=${id}`}>Reuse files and test plan</Button>{/if}
             {#if promotion}<Button href={`/campaigns/${encodeURIComponent(promotion.id)}`}>View campaign</Button>{/if}
+            {#if rollbackCampaign}<Button href={`/campaigns/${encodeURIComponent(rollbackCampaign.id)}`}>View rollback campaign</Button>{/if}
         </div>
         {#if promotion}<p class="muted">{promotion.pending} servers awaiting activation · {promotion.failed} need attention. Promoting a version does not mean it is already running everywhere.</p>{/if}
         {#if flash}<p role="alert" class="ui-error">{flash}</p>{/if}
@@ -166,7 +169,7 @@
     {#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if}
     {#if preview}
         <p><strong>{preview.servers.filter(server => server.tested).length} tested servers</strong> and <strong>{preview.servers.filter(server => !server.tested).length} additional eligible servers</strong>.</p>
-        <p>Files are installed when you confirm. Tested servers have already restarted. Additional servers restart when empty, then their health is checked. Eligibility is determined by each file's current scope.</p>
+        <p>Confirming queues the file changes. Each server waits until empty and restart capacity is available, then installs its files and restarts. Tested servers only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.</p>
         <p class="muted">Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed settings change, you must review again.</p>
         {#each preview.blockers as blocker}<p class="ui-error" role="alert">{blocker}</p>{/each}
         <div class="file-list">{#each preview.files as file}<details open><summary>{file.name} · {artifactVersionLabel(file)} · {file.servers.length} servers</summary><p><code>{file.path}</code> · {file.scope} · SHA {file.sha256.slice(0,12)}</p><ul>{#each file.servers as name}<li><ServerIdentity {name}/><span>{preview.servers.find(server => server.name === name)?.tested ? 'Already tested' : 'Additional target · restart when empty'}</span></li>{/each}</ul></details>{/each}</div>

src/routes/campaigns/[id]/+page.svelte

@@ -20,6 +20,7 @@
         id: number; server: string; verb: string; rel_path: string;
         sha256_before: string | null; sha256_after: string | null;
         size_before: number | null; size_after: number | null;
+        apply_state: 'pending' | 'applying' | 'applied' | 'cancelled';
     }
     interface ServerRow {
         deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number;
@@ -40,6 +41,8 @@
 
     let detail = $state<Detail | null>(null);
     let loadError = $state('');
+    let resuming = $state(false);
+    let resumeError = $state('');
     let request = 0;
     let diffOpen = $state(false);
     let diffError = $state('');
@@ -48,7 +51,8 @@
     let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null);
     let retryFile = $state<FileRow | null>(null);
     const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0);
-    const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
+    const withdrawn = $derived(detail?.servers.filter(server => ['skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
+    const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && !['done', 'skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
     const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
     const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => {
         const rank: Record<string, number> = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 };
@@ -73,6 +77,16 @@
 
     onMount(() => pollVisible(loadDetail, 5000, false));
 
+    async function resume() {
+        resuming = true; resumeError = '';
+        try {
+            const response = await fetch(`/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, { method: 'POST' });
+            if (!response.ok) throw new Error((await response.json()).error ?? 'Could not resume campaign.');
+            await loadDetail();
+        } catch (error) { resumeError = (error as Error).message; }
+        finally { resuming = false; }
+    }
+
     async function loadDiff(f: FileRow) {
         retryFile = f;
         const id = $page.params.id;
@@ -123,10 +137,21 @@
             {/snippet}
         </SectionHeader>
 
-        <section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
+        <section class="outcome" aria-label="Campaign outcome">
+            <h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2>
+            <p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}</p>
+            {#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>
+            {:else if withdrawn}<p>Withdrawn changes will not be activated by this campaign.</p>
+            {:else}<p>Every recorded server completed activation.</p>{/if}
+        </section>
 
         {#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Maintenance · <a href={`/maintenance/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
 
+        {#if detail.servers.some(server => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))}
+            <div><Button onclick={resume} loading={resuming} disabled={resuming}>Resume when empty</Button><p>Apply held changes and restart after the safety checks pass.</p></div>
+        {/if}
+        {#if resumeError}<p class="err" role="alert">{resumeError}</p>{/if}
+
         <Card padding="lg">
             <div class="meta-grid">
                 <div class="kv"><span class="k">Started</span><span class="v mono">{fmt(detail.deployment.started_at)}</span></div>
@@ -182,6 +207,7 @@
                             {#each files as f (f.id)}
                                 <button class="file-row" onclick={() => loadDiff(f)}>
                                     <Badge variant={verbVariant(f.verb)} size="sm" uppercase>{f.verb}</Badge>
+                                    {#if f.apply_state !== 'applied'}<Badge variant="neutral" size="sm">{f.apply_state === 'pending' ? 'Queued' : f.apply_state === 'applying' ? 'Needs inspection' : 'Cancelled'}</Badge>{/if}
                                     <span class="path">{f.rel_path}</span>
                                     <span class="sizes">
                                         {#if f.size_before !== null}{f.size_before} B{/if}