Defer server file changes until guarded restart activation

Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.

src/routes/api/betas/[id]/+server.ts

@@ -33,7 +33,9 @@ export async function GET({ params }: { params: { id: string } }) {
     const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
     const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined;
     const promotion = campaign ? { ...campaign,
-        pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n,
+        pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled','skipped')").get(campaign.id) as { n: number }).n,
         failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
-    return json({ beta, scripts, servers, events, test: testView(id), promotion });
+    const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1")
+        .get(id) as { id: string } | undefined;
+    return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null });
 }

src/routes/api/betas/[id]/retry/+server.ts

@@ -3,6 +3,8 @@ import { db } from '$lib/server/db/index.js';
 import { emit, type BetaStage } from '$lib/server/pipeline/bus.js';
 import { storedPlan } from '$lib/server/fleet/beta-tests.js';
 import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
+import { deployBetaServers, getBeta, isActiveBetaServer } from '$lib/server/fleet/beta.js';
+import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
 
 function parseId(raw: string): number | null {
     const n = Number.parseInt(raw, 10);
@@ -14,18 +16,33 @@ function parseId(raw: string): number | null {
 export async function POST({ params, request }: { params: { id: string }; request: Request }) {
     const betaId = parseId(params.id);
     if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
+    const beta = getBeta(betaId);
+    if (!beta || !['running', 'ready-promote'].includes(beta.state)) return json({ ok: false, error: 'Beta is closed or missing' }, { status: 409 });
     if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 });
     const body = (await request.json().catch(() => ({}))) as { server?: string };
     if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
 
     const row = db()
-        .prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
+        .prepare('SELECT stage, deployed_at, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
         .get(betaId, body.server) as
-        | { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
+        | { stage: string; deployed_at: string | null; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
         | undefined;
     if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 });
-    if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
+    if (!isActiveBetaServer(betaId, body.server)) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
     if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 });
+    if (stagedFileChanges('beta', String(betaId), body.server).some(change => change.state === 'applying')) {
+        return json({ ok: false, error: 'A file write was interrupted. Inspect the server and roll back before opening a fresh beta.' }, { status: 409 });
+    }
+
+    // Preparation may have failed before any bytes were queued (for example,
+    // an overlay without a baseline). Retrying must prepare its payload first.
+    if (!row.deployed_at && !row.restarted_at) {
+        try { await deployBetaServers(betaId, [body.server]); }
+        catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 409 }); }
+        const prepared = db().prepare('SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?')
+            .get(betaId, body.server) as { stage: string; error: string | null };
+        if (prepared.stage === 'failed') return json({ ok: false, error: prepared.error }, { status: 409 });
+    }
 
     // Best-effort resume point: if we already got past restart, go back to
     // health check; otherwise back to waiting for the server to be empty.

src/routes/api/betas/[id]/test/+server.ts

@@ -1,7 +1,8 @@
 import { json } from '@sveltejs/kit';
 import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
 import { db } from '$lib/server/db/index.js';
-import { getBeta } from '$lib/server/fleet/beta.js';
+import { deployBetaServers, getBeta } from '$lib/server/fleet/beta.js';
+import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
 import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
 import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
 import { parseBetaPlan } from '$lib/beta-plan.js';
@@ -52,7 +53,16 @@ export async function POST({ params, request }: { params: { id: string }; reques
             // Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
             // operators open another beta if the overall testing window has expired.
             if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
-            if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta');
+            if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
+                throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
+            }
+            const prepare = db().prepare("SELECT server FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL AND restarted_at IS NULL")
+                .all(id) as { server: string }[];
+            if (prepare.length) {
+                await deployBetaServers(id, prepare.map(row => row.server));
+                const failed = db().prepare("SELECT error FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL").get(id) as { error: string | null } | undefined;
+                if (failed) throw new Error(failed.error ?? 'Could not prepare beta files');
+            }
             db().transaction(() => {
                 const servers = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ?').all(id) as { server: string }[];
                 for (const { server } of servers) releaseRestartRequest(server, { kind: 'beta', id: String(id) });

src/routes/api/campaigns/[id]/+server.ts

@@ -17,7 +17,8 @@ export async function GET({ params }: { params: { id: string } }) {
         .prepare(
             `SELECT id, server, verb, rel_path, sha256_before, sha256_after,
                     LENGTH(content_before) AS size_before,
-                    LENGTH(content_after)  AS size_after
+                    LENGTH(content_after)  AS size_after,
+                    COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state
              FROM deployment_files
              WHERE deployment_id = ?
              ORDER BY server, rel_path`

src/routes/api/campaigns/[id]/resume/+server.ts

@@ -0,0 +1,25 @@
+import { json } from '@sveltejs/kit';
+import { db } from '$lib/server/db/index.js';
+import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
+import { kick } from '$lib/server/pipeline/worker.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    const changed = db().transaction(() => {
+        const rows = db().prepare(`SELECT ds.server, d.state FROM deployment_servers ds
+            LEFT JOIN server_deployments d ON d.id = ds.provisioning_id
+            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`)
+            .all(params.id) as { server: string; state: string | null }[];
+        for (const row of rows) {
+            if (row.state && ['failed','interrupted','cancelled'].includes(row.state)) {
+                releaseRestartRequest(row.server, { kind: 'campaign', id: params.id });
+            }
+        }
+        const result = db().prepare(`UPDATE deployment_servers SET restart_state = 'pending', error = NULL
+            WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`).run(params.id);
+        if (result.changes) db().prepare('UPDATE deployments SET finished_at = NULL WHERE id = ?').run(params.id);
+        return result.changes;
+    })();
+    if (!changed) return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 });
+    void kick().catch(err => console.error('[campaign] activation kick failed:', err));
+    return json({ ok: true }, { status: 202 });
+}