diff --git a/src/lib/operation-flow.ts b/src/lib/operation-flow.ts new file mode 100644 index 0000000..b832a12 --- /dev/null +++ b/src/lib/operation-flow.ts @@ -0,0 +1,41 @@ +export interface ActivityEvent { id: number; created_at: string; message: string } +export interface OperationStep { label: string; state: 'pending' | 'current' | 'done' | 'failed' } + +interface BetaServer { + stage: string; deployed_at: string | null; restarted_at: string | null; + healthy_at: string | null; match_seen_at: string | null; +} + +export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number }, servers: BetaServer[]): OperationStep[] { + const stages = ['deploying', 'awaiting-empty', 'restarting']; + const labels = ['Deploy artifacts', 'Wait for empty servers', 'Restart servers']; + if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); } + if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); } + stages.push('ready'); labels.push('Review and promote'); + const positions = servers.map(s => { + if (s.stage !== 'failed') return Math.max(0, stages.indexOf(s.stage)); + if (!s.deployed_at) return 0; + if (!s.restarted_at) return 2; + if (beta.wait_for_health && !s.healthy_at) return stages.indexOf('awaiting-health'); + if (beta.wait_for_match && !s.match_seen_at) return stages.indexOf('awaiting-match'); + return stages.length - 1; + }); + const current = beta.state === 'promoted' ? stages.length + : beta.state === 'ready-promote' ? stages.length - 1 + : positions.length ? Math.min(...positions) : 0; + return labels.map((label, i) => ({ label, state: i < current ? 'done' : i === current + ? beta.state === 'rolled-back' ? 'pending' : servers.some((s, j) => s.stage === 'failed' && positions[j] === i) ? 'failed' : 'current' + : 'pending' })); +} + +export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] { + const positions = servers.map(s => s.restart_state === 'done' ? 4 + : s.restart_state === 'failed' && !s.files_changed ? 0 + : s.restarted_at ? 3 : 1); + const current = positions.length ? Math.min(...positions) : finished ? 4 : 0; + return ['Apply file changes', 'Wait for empty servers', 'Restart servers', 'Verify health'].map((label, i) => ({ + label, state: i < current ? 'done' : i !== current ? 'pending' + : servers.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed' + : servers.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current' + })); +} diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index cca9016..e657553 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -286,3 +286,96 @@ CREATE TABLE IF NOT EXISTS server_creation_jobs ( map_slug TEXT NOT NULL, expected_map TEXT NOT NULL ); + +-- Persist state transitions atomically, including changes made by background workers. +-- Polls with unchanged status/error do not produce duplicate activity entries. +CREATE TABLE IF NOT EXISTS beta_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + message TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_beta_events ON beta_events(beta_id, id); +CREATE TRIGGER IF NOT EXISTS beta_activity_open AFTER INSERT ON betas +BEGIN + INSERT INTO beta_events (beta_id, created_at, message) + VALUES (NEW.id, NEW.opened_at, 'Beta opened'); +END; + +CREATE TABLE IF NOT EXISTS campaign_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + message TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_campaign_events ON campaign_events(deployment_id, id); +CREATE TRIGGER IF NOT EXISTS campaign_activity_file AFTER INSERT ON deployment_files +BEGIN + INSERT INTO campaign_events (deployment_id, message) + VALUES (NEW.deployment_id, NEW.server || ': ' || NEW.verb || ' ' || NEW.rel_path); +END; +CREATE TRIGGER IF NOT EXISTS campaign_activity_open AFTER INSERT ON deployments +BEGIN + INSERT INTO campaign_events (deployment_id, created_at, message) + VALUES (NEW.id, NEW.started_at, 'Campaign started: ' || NEW.action); +END; + +CREATE TRIGGER IF NOT EXISTS beta_activity_state AFTER UPDATE OF state ON betas +WHEN OLD.state IS NOT NEW.state +BEGIN + INSERT INTO beta_events (beta_id, message) VALUES (NEW.id, CASE NEW.state + WHEN 'ready-promote' THEN 'All beta servers are ready for review and promotion' + WHEN 'promoted' THEN 'Beta promoted' + WHEN 'rolled-back' THEN 'Beta rolled back' + ELSE 'Beta state: ' || NEW.state END); +END; +CREATE TRIGGER IF NOT EXISTS campaign_activity_finish AFTER UPDATE OF finished_at ON deployments +WHEN OLD.finished_at IS NULL AND NEW.finished_at IS NOT NULL +BEGIN + INSERT INTO campaign_events (deployment_id, created_at, message) + VALUES (NEW.id, NEW.finished_at, 'Campaign processing finished; see server statuses for held or manual work'); +END; + +CREATE TRIGGER IF NOT EXISTS beta_activity_server_insert AFTER INSERT ON beta_servers + +BEGIN + INSERT INTO beta_events (beta_id, message) VALUES (NEW.beta_id, NEW.server || ': ' || CASE WHEN NEW.stage = 'deploying' THEN 'Deploying artifacts' + WHEN NEW.stage = 'awaiting-empty' THEN 'Waiting for an empty server' + WHEN NEW.stage = 'restarting' THEN 'Restarting server' + WHEN NEW.stage = 'awaiting-health' THEN 'Verifying health' + WHEN NEW.stage = 'awaiting-match' THEN 'Waiting for a completed match' + WHEN NEW.stage = 'ready' THEN 'Ready for promotion' + WHEN NEW.stage = 'failed' THEN 'Failed' ELSE NEW.stage END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END); +END; + +CREATE TRIGGER IF NOT EXISTS beta_activity_server_update AFTER UPDATE ON beta_servers +WHEN OLD.stage IS NOT NEW.stage OR OLD.error IS NOT NEW.error OR OLD.restarted_at IS NOT NEW.restarted_at +BEGIN + INSERT INTO beta_events (beta_id, message) VALUES (NEW.beta_id, NEW.server || ': ' || CASE WHEN NEW.stage = 'deploying' THEN 'Deploying artifacts' + WHEN NEW.stage = 'awaiting-empty' THEN 'Waiting for an empty server' + WHEN NEW.stage = 'restarting' THEN 'Restarting server' + WHEN NEW.stage = 'awaiting-health' THEN 'Verifying health' + WHEN NEW.stage = 'awaiting-match' THEN 'Waiting for a completed match' + WHEN NEW.stage = 'ready' THEN 'Ready for promotion' + WHEN NEW.stage = 'failed' THEN 'Failed' ELSE NEW.stage END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END); +END; + +CREATE TRIGGER IF NOT EXISTS campaign_activity_server_insert AFTER INSERT ON deployment_servers + +BEGIN + INSERT INTO campaign_events (deployment_id, message) VALUES (NEW.deployment_id, NEW.server || ': ' || CASE WHEN NEW.restart_state = 'pending' AND NEW.restarted_at IS NOT NULL THEN 'Restart requested; verifying health' WHEN NEW.restart_state = 'pending' THEN 'Waiting for an empty server or restart capacity' + WHEN NEW.restart_state = 'done' THEN 'Server complete' + WHEN NEW.restart_state = 'failed' THEN 'Failed' + WHEN NEW.restart_state = 'manual' THEN 'Manual restart required' + WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END); +END; + +CREATE TRIGGER IF NOT EXISTS campaign_activity_server_update AFTER UPDATE ON deployment_servers +WHEN OLD.restart_state IS NOT NEW.restart_state OR OLD.error IS NOT NEW.error OR OLD.restarted_at IS NOT NEW.restarted_at +BEGIN + INSERT INTO campaign_events (deployment_id, message) VALUES (NEW.deployment_id, NEW.server || ': ' || CASE WHEN NEW.restart_state = 'pending' AND NEW.restarted_at IS NOT NULL THEN 'Restart requested; verifying health' WHEN NEW.restart_state = 'pending' THEN 'Waiting for an empty server or restart capacity' + WHEN NEW.restart_state = 'done' THEN 'Server complete' + WHEN NEW.restart_state = 'failed' THEN 'Failed' + WHEN NEW.restart_state = 'manual' THEN 'Manual restart required' + WHEN NEW.restart_state = 'held' THEN 'Changes held' ELSE NEW.restart_state END || CASE WHEN NEW.error IS NOT NULL THEN ' — ' || NEW.error ELSE '' END); +END; diff --git a/src/lib/ui/components/OperationHistory.svelte b/src/lib/ui/components/OperationHistory.svelte new file mode 100644 index 0000000..7e0072f --- /dev/null +++ b/src/lib/ui/components/OperationHistory.svelte @@ -0,0 +1,47 @@ + + +
+

Operation flow

+
    + {#each steps as step, index} +
  1. + {index + 1}{step.label} + ({step.state}) +
  2. + {/each} +
+
+
+

Activity

{#if active}Updates automatically{/if}
+ {#if events.length} +
    + {#each events as event (event.id)} +
  1. {event.message}

  2. + {/each} +
+ {:else} +

This operation predates detailed activity tracking. Its recorded status and timestamps are shown on this page.

+ {/if} +
+ + diff --git a/src/routes/api/betas/[id]/+server.ts b/src/routes/api/betas/[id]/+server.ts index b9cae80..859e302 100644 --- a/src/routes/api/betas/[id]/+server.ts +++ b/src/routes/api/betas/[id]/+server.ts @@ -1,4 +1,5 @@ import { json } from '@sveltejs/kit'; +import { db } from '$lib/server/db/index.js'; import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js'; import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; @@ -28,5 +29,6 @@ export async function GET({ params }: { params: { id: string } }) { }); const servers = listBetaServers(id); - return json({ beta, scripts, servers }); + const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id); + return json({ beta, scripts, servers, events }); } diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 7aed1fc..bbbb4fe 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -38,5 +38,6 @@ export async function GET({ params }: { params: { id: string } }) { ) .all(params.id); - return json({ deployment: dep, servers, files, scripts }); + const events = db().prepare('SELECT id, created_at, message FROM campaign_events WHERE deployment_id = ? ORDER BY id').all(params.id); + return json({ deployment: dep, servers, files, scripts, events }); } diff --git a/src/routes/betas/[id]/+page.svelte b/src/routes/betas/[id]/+page.svelte index 60dc5fb..5510f4b 100644 --- a/src/routes/betas/[id]/+page.svelte +++ b/src/routes/betas/[id]/+page.svelte @@ -7,6 +7,8 @@ import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; + import OperationHistory from '$ui/components/OperationHistory.svelte'; + import { betaFlow, type ActivityEvent } from '$lib/operation-flow.js'; interface BetaRow { id: number; script_id: number; script_version_id: number; @@ -31,6 +33,9 @@ let beta = $state(null); let scripts = $state([]); let servers = $state([]); + let events = $state([]); + let loadError = $state(''); + let request = 0; let loading = $state(true); let flash = $state(null); let acting = $state(null); @@ -38,18 +43,24 @@ let id = $derived(Number.parseInt($page.params.id ?? '0', 10)); async function load(): Promise { - const res = await fetch(`/api/betas/${id}`); - if (!res.ok) { loading = false; return; } - const body = (await res.json()) as { beta: BetaRow; scripts: BetaScript[]; servers: BetaServerRow[]; }; - beta = body.beta; - scripts = body.scripts; - servers = body.servers; - loading = false; + const current = ++request; + const requestedId = id; + try { + const res = await fetch(`/api/betas/${requestedId}`); + if (!res.ok) throw new Error(res.status === 404 ? 'Beta not found.' : 'Could not refresh beta.'); + const body = await res.json(); + if (current !== request || requestedId !== id) return; + beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; + loadError = ''; + } catch (err) { + if (current === request && requestedId === id) loadError = (err as Error).message; + } finally { if (current === request && requestedId === id) loading = false; } } + $effect(() => { id; beta = null; loading = true; loadError = ''; void load(); }); + let poll: ReturnType | null = null; onMount(() => { - load(); poll = setInterval(load, 5000); }); onDestroy(() => { if (poll) clearInterval(poll); }); @@ -106,6 +117,9 @@ } +Beta #{id} | Electric Cherry +{#if loadError}{/if} + {#if loading}
Loading…
{:else if !beta} @@ -122,6 +136,15 @@ {/snippet} + +
+
Started{new Date(b.opened_at).toLocaleString()}
+
Finished{b.closed_at ? new Date(b.closed_at).toLocaleString() : 'In progress'}
+
+
+ + +
@@ -173,9 +196,9 @@ {bs.stage}
Deploy - Empty + Empty Restart - Healthy + {#if b.wait_for_health}Healthy{/if} {#if b.wait_for_match} Match {/if} diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 43b094d..68764fc 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -8,6 +8,8 @@ import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; + import OperationHistory from '$ui/components/OperationHistory.svelte'; + import { campaignFlow, type ActivityEvent } from '$lib/operation-flow.js'; interface FileRow { id: number; server: string; verb: string; rel_path: string; @@ -27,20 +29,32 @@ }; servers: ServerRow[]; files: FileRow[]; + events: ActivityEvent[]; } let detail = $state(null); + let loadError = $state(''); + let request = 0; let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null); async function loadDetail(): Promise { const id = $page.params.id; - const res = await fetch(`/api/campaigns/${id}`); - detail = (await res.json()) as Detail; + const current = ++request; + try { + const res = await fetch(`/api/campaigns/${encodeURIComponent(id ?? '')}`); + if (!res.ok) throw new Error(res.status === 404 ? 'Campaign not found.' : 'Could not refresh campaign.'); + const body = await res.json() as Detail; + if (current !== request || id !== $page.params.id) return; + detail = body; loadError = ''; + } catch (err) { + if (current === request && id === $page.params.id) loadError = (err as Error).message; + } } + $effect(() => { $page.params.id; detail = null; openFile = null; loadError = ''; void loadDetail(); }); + let poll: ReturnType | null = null; onMount(() => { - loadDetail(); poll = setInterval(loadDetail, 5000); }); onDestroy(() => { @@ -86,6 +100,9 @@ } +Campaign | Electric Cherry +{#if loadError}{/if} + {#if detail} {@const action = detail.deployment.action}
@@ -107,6 +124,12 @@
+ s.restart_state === 'pending' || s.restart_state === 'failed')} + /> +

Servers

@@ -189,7 +212,7 @@
{/if}
-{:else} +{:else if !loadError}
Loading…
{/if} diff --git a/tests/unit/operation-history.test.ts b/tests/unit/operation-history.test.ts new file mode 100644 index 0000000..423fff3 --- /dev/null +++ b/tests/unit/operation-history.test.ts @@ -0,0 +1,66 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { betaFlow, campaignFlow } from '../../src/lib/operation-flow.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); }); +afterEach(async () => { await fixture.cleanup(); }); + +describe('persistent activity', () => { + it('keeps beta failures and retries across reopening without duplicating polls', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const handle = db(); + handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run(); + handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run(); + handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run(); + handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'deploying')").run(); + handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed' WHERE beta_id = 1").run(); + handle.prepare("UPDATE beta_servers SET stage = 'failed', error = 'write failed', updated_at = 'later' WHERE beta_id = 1").run(); + handle.prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL WHERE beta_id = 1").run(); + handle.prepare("UPDATE betas SET state = 'rolled-back' WHERE id = 1").run(); + const before = handle.prepare('SELECT * FROM beta_events ORDER BY id').all(); + expect(before).toHaveLength(5); + expect(before).toEqual(expect.arrayContaining([expect.objectContaining({ message: 'server: Failed — write failed' })])); + closeDb(); + expect(db().prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(before); + }); + + it('records campaign files, restart and health milestones, and cascades cleanup', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const handle = db(); + handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')").run(); + handle.prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')").run(); + handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')").run(); + handle.prepare("UPDATE deployment_servers SET attempts = attempts + 1, checked_at = 'later'").run(); + expect(handle.prepare('SELECT * FROM campaign_events').all()).toHaveLength(3); + handle.prepare("UPDATE deployment_servers SET restarted_at = '2026-09-17T00:01:00Z'").run(); + handle.prepare("UPDATE deployment_servers SET restart_state = 'done'").run(); + handle.prepare("UPDATE deployments SET finished_at = '2026-09-17T00:02:00Z'").run(); + const messages = (handle.prepare('SELECT message FROM campaign_events ORDER BY id').all() as { message: string }[]).map(e => e.message); + expect(messages).toHaveLength(6); + expect(messages).toContain('server: Restart requested; verifying health'); + expect(messages).toContain('server: Server complete'); + handle.prepare("DELETE FROM deployments WHERE id = 'campaign'").run(); + expect(handle.prepare('SELECT * FROM campaign_events').all()).toEqual([]); + }); +}); + +describe('operation progress', () => { + const server = { stage: 'awaiting-empty', deployed_at: 'now', restarted_at: null, healthy_at: null, match_seen_at: null }; + it('uses the slowest beta server and omits disabled gates', () => { + const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [server, { ...server, stage: 'ready' }]); + expect(flow.map(s => s.label)).toEqual(['Deploy artifacts', 'Wait for empty servers', 'Restart servers', 'Review and promote']); + expect(flow.map(s => s.state)).toEqual(['done', 'current', 'pending', 'pending']); + }); + it('does not mark a failed deployment as completed', () => { + expect(betaFlow({ state: 'running', wait_for_health: 1, wait_for_match: 1 }, [{ ...server, stage: 'failed', deployed_at: null }])[0].state).toBe('failed'); + expect(campaignFlow([{ restart_state: 'failed', files_changed: 0, restarted_at: null }], false)[0].state).toBe('failed'); + }); + it('requires health verification after campaign restart and preserves manual work', () => { + expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'current']); + expect(campaignFlow([{ restart_state: 'manual', files_changed: 1, restarted_at: null }], true).some(s => s.state === 'current')).toBe(false); + expect(campaignFlow([{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], true).every(s => s.state === 'done')).toBe(true); + }); +});