Defer server file changes until guarded restart activation

Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.

tests/unit/campaign-resume.test.ts

@@ -0,0 +1,112 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+const server = 't5_zombie_theater_1';
+const campaign = 'held-rollback';
+const before = Buffer.from('current live bytes');
+const restored = Buffer.from('restored version');
+const destination = () => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc');
+
+beforeEach(async () => {
+    vi.resetModules();
+    vi.clearAllMocks();
+    fixture = await newFixture([server]);
+    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+    await seedServerRegistry();
+});
+afterEach(async () => { await fixture.cleanup(); });
+
+async function heldRollback(state = 'held') {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+    await writeFile(destination(), before);
+    const now = new Date().toISOString();
+    db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)")
+        .run(campaign, now, now);
+    db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error)
+        VALUES (?, ?, 1, ?, 'Waiting for manual activation')`).run(campaign, server, state);
+    queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: 'rollback.gsc',
+        content: restored, inventorySource: 'rollback:beta-1' });
+}
+
+async function resume(id = campaign) {
+    const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
+    return POST({ params: { id } });
+}
+
+describe('campaign resume API', () => {
+    it.each(['held', 'manual'])('queues %s rollback changes without touching live files', async (state) => {
+        await heldRollback(state);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { kick } = await import('../../src/lib/server/pipeline/worker.js');
+        const response = await resume();
+        expect(response.status).toBe(202);
+        expect(await response.json()).toEqual({ ok: true });
+        expect(db().prepare('SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
+            .toEqual({ restart_state: 'pending', error: null, provisioning_id: null });
+        expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)).toEqual({ finished_at: null });
+        expect(db().prepare('SELECT state, content FROM staged_file_changes WHERE source_id = ?').get(campaign))
+            .toEqual({ state: 'pending', content: restored });
+        expect(await readFile(destination())).toEqual(before);
+        expect(kick).toHaveBeenCalledTimes(1);
+        expect((await resume()).status).toBe(409);
+        expect(kick).toHaveBeenCalledTimes(1);
+    });
+
+    it.each(['cancelled', 'failed', 'interrupted'])('releases an old %s operation so the worker creates fresh restart intent', async (state) => {
+        await heldRollback();
+        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const old = requestRestart(server, { kind: 'campaign', id: campaign });
+        db().prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
+            .run(state, new Date().toISOString(), old.id);
+        expect((await resume()).status).toBe(202);
+        expect(db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
+            .toEqual({ provisioning_id: null });
+        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+        await sweep();
+        const next = requestRestart(server, { kind: 'campaign', id: campaign });
+        expect(next.id).not.toBe(old.id);
+        expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
+        expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)).toEqual({ state });
+        expect(db().prepare('SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid').all(campaign))
+            .toEqual([{ operation_id: old.id }, { operation_id: next.id }]);
+        expect(await readFile(destination())).toEqual(before);
+    });
+
+    it('retains an existing waiting operation when its held campaign resumes', async () => {
+        await heldRollback();
+        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+        const old = requestRestart(server, { kind: 'campaign', id: campaign });
+        expect((await resume()).status).toBe(202);
+        expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id);
+    });
+
+    it.each(['missing', 'done', 'deleted'])('rejects a campaign with %s changes that cannot resume', async (condition) => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { kick } = await import('../../src/lib/server/pipeline/worker.js');
+        if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held');
+        if (condition === 'deleted') {
+            db().prepare('DELETE FROM servers WHERE name = ?').run(server);
+            const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+            await seedServerRegistry(); // Reusing its name must not resume old work.
+        }
+        const response = await resume();
+        expect(response.status).toBe(409);
+        expect(await response.json()).toEqual({ error: 'This campaign has no held changes to resume.' });
+        expect(kick).not.toHaveBeenCalled();
+        expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
+        if (condition !== 'missing') {
+            expect(await readFile(destination())).toEqual(before);
+            expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign))
+                .toEqual({ restart_state: condition === 'done' ? 'done' : 'held' });
+            expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign))
+                .toMatchObject({ finished_at: expect.any(String) });
+        }
+    });
+});