josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add operation flows and activity logs for betas and campaigns

Browse Source
This commit is contained in:
josh
2026-09-17 19:34:26 -04:00
parent 9dcf4ac1d3
commit 5b5266da3a
8 changed files with 312 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/operation-flow.ts
+41
View File
@@ -0,0 +1,41 @@
export interface ActivityEvent { id: number; created_at: string; message: string }
export interface OperationStep { label: string; state: 'pending' | 'current' | 'done' | 'failed' }
interface BetaServer {
stage: string; deployed_at: string | null; restarted_at: string | null;
healthy_at: string | null; match_seen_at: string | null;
}
export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number }, servers: BetaServer[]): OperationStep[] {
const stages = ['deploying', 'awaiting-empty', 'restarting'];
const labels = ['Deploy artifacts', 'Wait for empty servers', 'Restart servers'];
if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); }
if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); }
stages.push('ready'); labels.push('Review and promote');
const positions = servers.map(s => {
if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage));
if (!s.deployed_at) return 0;
if (!s.restarted_at) return 2;
if (beta.wait_for_health && !s.healthy_at) return stages.indexOf('awaiting-health');
if (beta.wait_for_match && !s.match_seen_at) return stages.indexOf('awaiting-match');
return stages.length - 1;
});
const current = beta.state === 'promoted' ? stages.length
: beta.state === 'ready-promote' ? stages.length - 1
: positions.length ? Math.min(...positions) : 0;
return labels.map((label, i) => ({ label, state: i < current ? 'done' : i === current
? beta.state === 'rolled-back' ? 'pending' : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) ? 'failed' : 'current'
: 'pending' }));
}
export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] {
const positions = servers.map(s => s.restart_state === 'done' ? 4
: s.restart_state === 'failed' && !s.files_changed ? 0
: s.restarted_at ? 3 : 1);
const current = positions.length ? Math.min(...positions) : finished ? 4 : 0;
return ['Apply file changes', 'Wait for empty servers', 'Restart servers', 'Verify health'].map((label, i) => ({
label, state: i < current ? 'done' : i !== current ? 'pending'
: servers.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: servers.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
}));
}
src/lib/server/db/schema.sql
+93
View File
@@ -286,3 +286,96 @@ CREATE TABLE IF NOT EXISTS server_creation_jobs (
map_slug TEXT NOT NULL,
expected_map TEXT NOT NULL
);
-- Persist state transitions atomically, including changes made by background workers.
-- Polls with unchanged status/error do not produce duplicate activity entries.
CREATE TABLE IF NOT EXISTS beta_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
message TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_beta_events ON beta_events(beta_id, id);
CREATE TRIGGER IF NOT EXISTS beta_activity_open AFTER INSERT ON betas
BEGIN
INSERT INTO beta_events (beta_id, created_at, message)
VALUES (NEW.id, NEW.opened_at, 'Beta opened');
END;
CREATE TABLE IF NOT EXISTS campaign_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
message TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_campaign_events ON campaign_events(deployment_id, id);
CREATE TRIGGER IF NOT EXISTS campaign_activity_file AFTER INSERT ON deployment_files
BEGIN
INSERT INTO campaign_events (deployment_id, message)
VALUES (NEW.deployment_id, NEW.server || ': ' || NEW.verb || ' ' || NEW.rel_path);
END;
CREATE TRIGGER IF NOT EXISTS campaign_activity_open AFTER INSERT ON deployments
BEGIN
INSERT INTO campaign_events (deployment_id, created_at, message)
VALUES (NEW.id, NEW.started_at, 'Campaign started: ' || NEW.action);
END;
CREATE TRIGGER IF NOT EXISTS beta_activity_state AFTER UPDATE OF state ON betas
WHEN OLD.state IS NOT NEW.state
BEGIN
INSERT INTO beta_events (beta_id, message) VALUES (NEW.id, CASE NEW.state
WHEN 'ready-promote' THEN 'All beta servers are ready for review and promotion'
WHEN 'promoted' THEN 'Beta promoted'
WHEN 'rolled-back' THEN 'Beta rolled back'
ELSE 'Beta state: ' || NEW.state END);
END;
CREATE TRIGGER IF NOT EXISTS campaign_activity_finish AFTER UPDATE OF finished_at ON deployments
WHEN OLD.finished_at IS NULL AND NEW.finished_at IS NOT NULL
BEGIN
INSERT INTO campaign_events (deployment_id, created_at, message)
VALUES (NEW.id, NEW.finished_at, 'Campaign processing finished; see server statuses for held or manual work');
END;
CREATE TRIGGER IF NOT EXISTS beta_activity_server_insert AFTER INSERT ON beta_servers
BEGIN
INSERT INTO beta_events (beta_id, message) VALUES (NEW.beta_id, NEW.server || ': ' || CASE WHEN NEW.stage = 'deploying' THEN 'Deploying artifacts'
WHEN NEW.stage = 'awaiting-empty' THEN 'Waiting for an empty server'
WHEN NEW.stage = 'restarting' THEN 'Restarting server'
WHEN NEW.stage = 'awaiting-health' THEN 'Verifying health'
WHEN NEW.stage = 'awaiting-match' THEN 'Waiting for a completed match'
WHEN NEW.stage = 'ready' THEN 'Ready for promotion'
WHEN NEW.stage = 'failed' THEN 'Failed' ELSE NEW.stage END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END);
END;
CREATE TRIGGER IF NOT EXISTS beta_activity_server_update AFTER UPDATE ON beta_servers
WHEN OLD.stage IS NOT NEW.stage OR OLD.error IS NOT NEW.error OR OLD.restarted_at IS NOT NEW.restarted_at
BEGIN
INSERT INTO beta_events (beta_id, message) VALUES (NEW.beta_id, NEW.server || ': ' || CASE WHEN NEW.stage = 'deploying' THEN 'Deploying artifacts'
WHEN NEW.stage = 'awaiting-empty' THEN 'Waiting for an empty server'
WHEN NEW.stage = 'restarting' THEN 'Restarting server'
WHEN NEW.stage = 'awaiting-health' THEN 'Verifying health'
WHEN NEW.stage = 'awaiting-match' THEN 'Waiting for a completed match'
WHEN NEW.stage = 'ready' THEN 'Ready for promotion'
WHEN NEW.stage = 'failed' THEN 'Failed' ELSE NEW.stage END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END);
END;
CREATE TRIGGER IF NOT EXISTS campaign_activity_server_insert AFTER INSERT ON deployment_servers
BEGIN
INSERT INTO campaign_events (deployment_id, message) VALUES (NEW.deployment_id, NEW.server || ': ' || CASE WHEN NEW.restart_state = 'pending' AND NEW.restarted_at IS NOT NULL THEN 'Restart requested; verifying health' WHEN NEW.restart_state = 'pending' THEN 'Waiting for an empty server or restart capacity'
WHEN NEW.restart_state = 'done' THEN 'Server complete'
WHEN NEW.restart_state = 'failed' THEN 'Failed'
WHEN NEW.restart_state = 'manual' THEN 'Manual restart required'
WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END);
END;
CREATE TRIGGER IF NOT EXISTS campaign_activity_server_update AFTER UPDATE ON deployment_servers
WHEN OLD.restart_state IS NOT NEW.restart_state OR OLD.error IS NOT NEW.error OR OLD.restarted_at IS NOT NEW.restarted_at
BEGIN
INSERT INTO campaign_events (deployment_id, message) VALUES (NEW.deployment_id, NEW.server || ': ' || CASE WHEN NEW.restart_state = 'pending' AND NEW.restarted_at IS NOT NULL THEN 'Restart requested; verifying health' WHEN NEW.restart_state = 'pending' THEN 'Waiting for an empty server or restart capacity'
WHEN NEW.restart_state = 'done' THEN 'Server complete'
WHEN NEW.restart_state = 'failed' THEN 'Failed'
WHEN NEW.restart_state = 'manual' THEN 'Manual restart required'
WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END);
END;
src/lib/ui/components/OperationHistory.svelte
+47
View File
@@ -0,0 +1,47 @@
<script lang="ts">
import type { ActivityEvent, OperationStep } from '$lib/operation-flow.js';
let { steps, events, active = false }: { steps: OperationStep[]; events: ActivityEvent[]; active?: boolean } = $props();
</script>
<section>
<h2>Operation flow</h2>
<ol class="flow">
{#each steps as step, index}
<li class={step.state} aria-current={step.state === 'current' ? 'step' : undefined}>
<span>{index + 1}</span>{step.label}
<span class="sr-only">({step.state})</span>
</li>
{/each}
</ol>
</section>
<section>
<div class="top"><h2>Activity</h2>{#if active}<span class="muted">Updates automatically</span>{/if}</div>
{#if events.length}
<ol class="timeline">
{#each events as event (event.id)}
<li><time datetime={event.created_at}>{new Date(event.created_at).toLocaleString()}</time><p>{event.message}</p></li>
{/each}
</ol>
{:else}
<p class="muted">This operation predates detailed activity tracking. Its recorded status and timestamps are shown on this page.</p>
{/if}
</section>
<style>
section { display: grid; gap: 16px; }
h2 { font-size: 18px; margin: 0; }
.top { display: flex; align-items: center; justify-content: space-between; gap: 16px; flex-wrap: wrap; }
time, .muted { color: var(--text-secondary); font-size: 13px; }
p { margin: 0; line-height: 1.6; }
.flow { display: flex; flex-wrap: wrap; gap: 12px; padding: 0; margin: 0; list-style: none; }
.flow li { display: flex; align-items: center; gap: 10px; background: var(--surface); border-radius: 8px; padding: 12px; font-size: 13px; border: 1px solid var(--border-subtle); color: var(--text-secondary); }
.flow span { color: var(--accent-primary-hover); font-family: var(--font-mono); }
.flow .done { color: var(--text-primary); }
.flow .current { color: var(--text-primary); border-color: var(--accent-primary); }
.flow .failed { color: var(--state-danger); border-color: var(--state-danger); }
.timeline { list-style: none; padding: 0 0 0 18px; margin: 0; border-left: 2px solid var(--border-strong); }
.timeline li { position: relative; padding: 0 0 24px 12px; overflow-wrap: anywhere; }
.timeline li::before { content: ''; position: absolute; left: -24px; top: 5px; width: 10px; height: 10px; border-radius: 50%; background: var(--accent-primary); }
.timeline p { margin-top: 6px; }
.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); }
</style>
src/routes/api/betas/[id]/+server.ts
+3 -1
View File
@@ -1,4 +1,5 @@
import { json } from '@sveltejs/kit';
import { db } from '$lib/server/db/index.js';
import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js';
import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
@@ -28,5 +29,6 @@ export async function GET({ params }: { params: { id: string } }) {
});
const servers = listBetaServers(id);
return json({ beta, scripts, servers });
const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
return json({ beta, scripts, servers, events });
}
src/routes/api/campaigns/[id]/+server.ts
+2 -1
View File
@@ -38,5 +38,6 @@ export async function GET({ params }: { params: { id: string } }) {
)
.all(params.id);
return json({ deployment: dep, servers, files, scripts });
const events = db().prepare('SELECT id, created_at, message FROM campaign_events WHERE deployment_id = ? ORDER BY id').all(params.id);
return json({ deployment: dep, servers, files, scripts, events });
}
src/routes/betas/[id]/+page.svelte
+33 -10
View File
@@ -7,6 +7,8 @@
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import OperationHistory from '$ui/components/OperationHistory.svelte';
import { betaFlow, type ActivityEvent } from '$lib/operation-flow.js';
interface BetaRow {
id: number; script_id: number; script_version_id: number;
@@ -31,6 +33,9 @@
let beta = $state<BetaRow | null>(null);
let scripts = $state<BetaScript[]>([]);
let servers = $state<BetaServerRow[]>([]);
let events = $state<ActivityEvent[]>([]);
let loadError = $state('');
let request = 0;
let loading = $state(true);
let flash = $state<string | null>(null);
let acting = $state<string | null>(null);
@@ -38,18 +43,24 @@
let id = $derived(Number.parseInt($page.params.id ?? '0', 10));
async function load(): Promise<void> {
const res = await fetch(`/api/betas/${id}`);
if (!res.ok) { loading = false; return; }
const body = (await res.json()) as { beta: BetaRow; scripts: BetaScript[]; servers: BetaServerRow[]; };
beta = body.beta;
scripts = body.scripts;
servers = body.servers;
loading = false;
const current = ++request;
const requestedId = id;
try {
const res = await fetch(`/api/betas/${requestedId}`);
if (!res.ok) throw new Error(res.status === 404 ? 'Beta not found.' : 'Could not refresh beta.');
const body = await res.json();
if (current !== request || requestedId !== id) return;
beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events;
loadError = '';
} catch (err) {
if (current === request && requestedId === id) loadError = (err as Error).message;
} finally { if (current === request && requestedId === id) loading = false; }
}
$effect(() => { id; beta = null; loading = true; loadError = ''; void load(); });
let poll: ReturnType<typeof setInterval> | null = null;
onMount(() => {
load();
poll = setInterval(load, 5000);
});
onDestroy(() => { if (poll) clearInterval(poll); });
@@ -106,6 +117,9 @@
}
</script>
<svelte:head><title>Beta #{id} | Electric Cherry</title></svelte:head>
{#if loadError}<p role="alert" class="err">{loadError} <button onclick={load}>Retry</button></p>{/if}
{#if loading}
<div class="loading">Loading…</div>
{:else if !beta}
@@ -122,6 +136,15 @@
{/snippet}
</SectionHeader>
<Card padding="lg">
<div class="opts">
<div class="opt"><span class="opt-k">Started</span><span class="opt-v">{new Date(b.opened_at).toLocaleString()}</span></div>
<div class="opt"><span class="opt-k">Finished</span><span class="opt-v">{b.closed_at ? new Date(b.closed_at).toLocaleString() : 'In progress'}</span></div>
</div>
</Card>
<OperationHistory steps={betaFlow(b, servers)} {events} active={b.state === 'running' || b.state === 'ready-promote'} />
<!-- Artifacts -->
<Card padding="lg">
<div class="panel-head">
@@ -173,9 +196,9 @@
<Badge variant={stageVariant(bs.stage)} size="sm" uppercase>{bs.stage}</Badge>
<div class="ticks" aria-label="pipeline progress">
<span class:on={bs.deployed_at}>Deploy</span>
<span class:on={bs.stage !== 'awaiting-empty' && bs.stage !== 'deploying'}>Empty</span>
<span class:on={bs.restarted_at || bs.stage === 'restarting'}>Empty</span>
<span class:on={bs.restarted_at}>Restart</span>
<span class:on={bs.healthy_at}>Healthy</span>
{#if b.wait_for_health}<span class:on={bs.healthy_at}>Healthy</span>{/if}
{#if b.wait_for_match}
<span class:on={bs.match_seen_at}>Match</span>
{/if}
src/routes/campaigns/[id]/+page.svelte
+27 -4
View File
@@ -8,6 +8,8 @@
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import OperationHistory from '$ui/components/OperationHistory.svelte';
import { campaignFlow, type ActivityEvent } from '$lib/operation-flow.js';
interface FileRow {
id: number; server: string; verb: string; rel_path: string;
@@ -27,20 +29,32 @@
};
servers: ServerRow[];
files: FileRow[];
events: ActivityEvent[];
}
let detail = $state<Detail | null>(null);
let loadError = $state('');
let request = 0;
let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null);
async function loadDetail(): Promise<void> {
const id = $page.params.id;
const res = await fetch(`/api/campaigns/${id}`);
detail = (await res.json()) as Detail;
const current = ++request;
try {
const res = await fetch(`/api/campaigns/${encodeURIComponent(id ?? '')}`);
if (!res.ok) throw new Error(res.status === 404 ? 'Campaign not found.' : 'Could not refresh campaign.');
const body = await res.json() as Detail;
if (current !== request || id !== $page.params.id) return;
detail = body; loadError = '';
} catch (err) {
if (current === request && id === $page.params.id) loadError = (err as Error).message;
}
}
$effect(() => { $page.params.id; detail = null; openFile = null; loadError = ''; void loadDetail(); });
let poll: ReturnType<typeof setInterval> | null = null;
onMount(() => {
loadDetail();
poll = setInterval(loadDetail, 5000);
});
onDestroy(() => {
@@ -86,6 +100,9 @@
}
</script>
<svelte:head><title>Campaign | Electric Cherry</title></svelte:head>
{#if loadError}<p role="alert" class="err">{loadError} <button onclick={loadDetail}>Retry</button></p>{/if}
{#if detail}
{@const action = detail.deployment.action}
<div class="page">
@@ -107,6 +124,12 @@
</div>
</Card>
<OperationHistory
steps={campaignFlow(detail.servers, Boolean(detail.deployment.finished_at))}
events={detail.events}
active={detail.servers.some(s => s.restart_state === 'pending' || s.restart_state === 'failed')}
/>
<section class="section">
<div class="section-head">
<h2>Servers</h2>
@@ -189,7 +212,7 @@
</div>
{/if}
</div>
{:else}
{:else if !loadError}
<div class="loading">Loading…</div>
{/if}
tests/unit/operation-history.test.ts
+66
View File
@@ -0,0 +1,66 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { betaFlow, campaignFlow } from '../../src/lib/operation-flow.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
afterEach(async () => { await fixture.cleanup(); });
describe('persistent activity', () => {
it('keeps beta failures and retries across reopening without duplicating polls', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const handle = db();
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run();
handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run();
handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run();
handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'deploying')").run();
handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed' WHERE beta_id = 1").run();
handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed', updated_at = 'later' WHERE beta_id = 1").run();
handle.prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL WHERE beta_id = 1").run();
handle.prepare("UPDATE betas SET state = 'rolled-back' WHERE id = 1").run();
const before = handle.prepare('SELECT * FROM beta_events ORDER BY id').all();
expect(before).toHaveLength(5);
expect(before).toEqual(expect.arrayContaining([expect.objectContaining({ message: 'server: Failed — write failed' })]));
closeDb();
expect(db().prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(before);
});
it('records campaign files, restart and health milestones, and cascades cleanup', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const handle = db();
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')").run();
handle.prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')").run();
handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')").run();
handle.prepare("UPDATE deployment_servers SET attempts = attempts + 1, checked_at = 'later'").run();
expect(handle.prepare('SELECT * FROM campaign_events').all()).toHaveLength(3);
handle.prepare("UPDATE deployment_servers SET restarted_at = '2026-09-17T00:01:00Z'").run();
handle.prepare("UPDATE deployment_servers SET restart_state = 'done'").run();
handle.prepare("UPDATE deployments SET finished_at = '2026-09-17T00:02:00Z'").run();
const messages = (handle.prepare('SELECT message FROM campaign_events ORDER BY id').all() as { message: string }[]).map(e => e.message);
expect(messages).toHaveLength(6);
expect(messages).toContain('server: Restart requested; verifying health');
expect(messages).toContain('server: Server complete');
handle.prepare("DELETE FROM deployments WHERE id = 'campaign'").run();
expect(handle.prepare('SELECT * FROM campaign_events').all()).toEqual([]);
});
});
describe('operation progress', () => {
const server = { stage: 'awaiting-empty', deployed_at: 'now', restarted_at: null, healthy_at: null, match_seen_at: null };
it('uses the slowest beta server and omits disabled gates', () => {
const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [server, { ...server, stage: 'ready' }]);
expect(flow.map(s => s.label)).toEqual(['Deploy artifacts', 'Wait for empty servers', 'Restart servers', 'Review and promote']);
expect(flow.map(s => s.state)).toEqual(['done', 'current', 'pending', 'pending']);
});
it('does not mark a failed deployment as completed', () => {
expect(betaFlow({ state: 'running', wait_for_health: 1, wait_for_match: 1 }, [{ ...server, stage: 'failed', deployed_at: null }])[0].state).toBe('failed');
expect(campaignFlow([{ restart_state: 'failed', files_changed: 0, restarted_at: null }], false)[0].state).toBe('failed');
});
it('requires health verification after campaign restart and preserves manual work', () => {
expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'current']);
expect(campaignFlow([{ restart_state: 'manual', files_changed: 1, restarted_at: null }], true).some(s => s.state === 'current')).toBe(false);
expect(campaignFlow([{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], true).every(s => s.state === 'done')).toBe(true);
});
});