From fb48f64de14e586274d8a6502fc31c4e96e42a1d Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 27 Sep 2026 23:06:16 -0400 Subject: [PATCH] Defer server file changes until guarded restart activation Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely. Fixes #28 Validation: 805 tests passed, Svelte checks clean, production build passed. --- README.md | 11 + docs/beta-test-plans.md | 6 +- src/lib/operation-flow.ts | 14 +- src/lib/server/db/schema.sql | 34 +- src/lib/server/fleet/beta.ts | 399 ++++++---------- src/lib/server/fleet/drift.ts | 55 ++- src/lib/server/fleet/rollback.ts | 79 ++-- src/lib/server/fleet/staged-files.ts | 155 ++++++ src/lib/server/pipeline/beta-plan-runner.ts | 17 +- src/lib/server/provisioning/requests.ts | 8 +- src/lib/server/provisioning/restarts.ts | 15 +- src/lib/server/provisioning/worker.ts | 11 +- src/lib/ui/components/BetaPlanBuilder.svelte | 2 +- src/lib/ui/status.ts | 2 +- src/routes/api/betas/[id]/+server.ts | 6 +- src/routes/api/betas/[id]/retry/+server.ts | 23 +- src/routes/api/betas/[id]/test/+server.ts | 14 +- src/routes/api/campaigns/[id]/+server.ts | 3 +- .../api/campaigns/[id]/resume/+server.ts | 25 + src/routes/betas/[id]/+page.svelte | 13 +- src/routes/campaigns/[id]/+page.svelte | 30 +- tests/helpers/staged-files.ts | 10 + tests/unit/beta-plans.test.ts | 12 +- tests/unit/beta-promotion.test.ts | 4 + tests/unit/beta-staging.test.ts | 232 +++++++++ tests/unit/campaign-resume.test.ts | 112 +++++ tests/unit/config-flow.test.ts | 15 +- tests/unit/create-server.test.ts | 6 +- tests/unit/deleted-server-guards.test.ts | 7 +- tests/unit/drift-kinds.test.ts | 123 +++-- tests/unit/map-config-flow.test.ts | 12 +- tests/unit/operation-history.test.ts | 2 +- tests/unit/rollback-staging.test.ts | 128 +++++ tests/unit/script-flow.test.ts | 12 +- tests/unit/staged-files.test.ts | 440 ++++++++++++++++++ tests/unit/t6-config-flow.test.ts | 18 +- 36 files changed, 1634 insertions(+), 421 deletions(-) create mode 100644 src/lib/server/fleet/staged-files.ts create mode 100644 src/routes/api/campaigns/[id]/resume/+server.ts create mode 100644 tests/helpers/staged-files.ts create mode 100644 tests/unit/beta-staging.test.ts create mode 100644 tests/unit/campaign-resume.test.ts create mode 100644 tests/unit/rollback-staging.test.ts create mode 100644 tests/unit/staged-files.test.ts diff --git a/README.md b/README.md index 235b420..350f9bf 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,17 @@ RCON parsing (fake UDP socket). Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility. +Beta deployments, promotions, drift repairs, and rollbacks stage their file changes +in SQLite. Live server files and inventory remain unchanged while waiting for an +empty or stopped server, other operations, and rolling restart capacity. Provisioning +captures rollback snapshots and applies the staged changes immediately before +requesting the restart. Failed file activation blocks that restart. Queued changes +survive application restarts; cancelling an unapplied beta discards its pending +changes. Manual beta rollback holds both file restoration and restart until +**Resume when empty** is selected on its rollback campaign. +Finish pending file campaigns before opening a beta on their servers; this keeps +an older queued change from overwriting the version tested by the new beta. + Deleting a server preserves its name, recorded beta progress and test evidence, campaign results and file diffs, provisioning and maintenance history, and match records. Historical entries show **Deleted** and omit live server links and diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md index 7554fcc..9db8107 100644 --- a/docs/beta-test-plans.md +++ b/docs/beta-test-plans.md @@ -180,8 +180,10 @@ servers. Missing evidence at the deadline produces **inconclusive**. Health monitoring continues after a pass until promotion. For failures, select hold as failed, pause for investigation, or automatic -rollback. Rollback restores only files touched by the beta. Restarts can wait -for empty servers or remain manual. Promotion is always manual. +rollback. Rollback queues restoration only for files touched by the beta. +Restoration and restart both wait for an empty server and restart capacity. +The manual option holds those changes in a rollback campaign until you select +**Resume when empty**. Promotion is always manual. Pause holds evaluation and wave advancement. Resume retains completed progress and the original deadline, but starts new timing intervals. A fresh attempt resets diff --git a/src/lib/operation-flow.ts b/src/lib/operation-flow.ts index 87468af..693b64e 100644 --- a/src/lib/operation-flow.ts +++ b/src/lib/operation-flow.ts @@ -8,7 +8,7 @@ interface BetaServer { export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] { const stages = ['deploying', 'awaiting-empty', 'restarting']; - const labels = ['Deploy artifacts', 'Wait for empty servers', 'Restart servers']; + const labels = ['Stage file changes', 'Wait for empty servers', 'Apply files and restart']; if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); } if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); } if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); } @@ -30,13 +30,13 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo } export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] { - const positions = servers.map(s => s.restart_state === 'done' ? 4 - : s.restart_state === 'failed' && !s.files_changed ? 0 - : s.restarted_at ? 3 : 1); + const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state)); + const positions = active.map(s => s.restart_state === 'done' ? 4 + : s.restarted_at ? 3 : 0); const current = positions.length ? Math.min(...positions) : finished ? 4 : 0; - return ['Apply file changes', 'Wait for empty servers', 'Restart servers', 'Verify health'].map((label, i) => ({ + return ['Wait for empty servers and restart capacity', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({ label, state: i < current ? 'done' : i !== current ? 'pending' - : servers.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed' - : servers.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current' + : active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed' + : active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current' })); } diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 047c80b..fa76753 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -343,6 +343,35 @@ CREATE TABLE IF NOT EXISTS deployment_files ( CREATE INDEX IF NOT EXISTS idx_deployment_files_dep ON deployment_files(deployment_id); +-- File payloads stay in SQLite until Provisioning has admitted their restart. +-- Existing campaigns/betas without rows here already wrote their files in the +-- previous release; they remain restart-only work during upgrade. +CREATE TABLE IF NOT EXISTS staged_file_changes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_kind TEXT NOT NULL CHECK (source_kind IN ('campaign','beta')), + source_id TEXT NOT NULL, + server TEXT NOT NULL, + kind TEXT NOT NULL, + rel_path TEXT NOT NULL, + content BLOB, + script_version_id INTEGER, + inventory_source TEXT NOT NULL, + deployment_file_id INTEGER REFERENCES deployment_files(id) ON DELETE CASCADE, + preserve_before_image INTEGER NOT NULL DEFAULT 0, + check_before INTEGER NOT NULL DEFAULT 0, + expected_before_sha TEXT, + state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','applying','applied','cancelled')), + applied_at TEXT, + UNIQUE (source_kind, source_id, server, kind, rel_path) +); +CREATE INDEX IF NOT EXISTS idx_staged_files_source ON staged_file_changes(source_kind, source_id, server); +CREATE TRIGGER IF NOT EXISTS staged_files_campaign_delete AFTER DELETE ON deployments BEGIN + DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = OLD.id; +END; +CREATE TRIGGER IF NOT EXISTS staged_files_beta_delete AFTER DELETE ON betas BEGIN + DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = CAST(OLD.id AS TEXT); +END; + -- Per-script rows on aggregate promote/rollback deployments. For legacy -- single-script deployments this may also be backfilled from the deprecated -- deployments.script_id / script_version_id / prior_version_id columns. @@ -550,10 +579,11 @@ CREATE TABLE IF NOT EXISTS campaign_events ( message TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_campaign_events ON campaign_events(deployment_id, id); -CREATE TRIGGER IF NOT EXISTS campaign_activity_file AFTER INSERT ON deployment_files +DROP TRIGGER IF EXISTS campaign_activity_file; +CREATE TRIGGER campaign_activity_file AFTER INSERT ON deployment_files BEGIN INSERT INTO campaign_events (deployment_id, message) - VALUES (NEW.deployment_id, NEW.server || ': ' || NEW.verb || ' ' || NEW.rel_path); + VALUES (NEW.deployment_id, NEW.server || ': recorded file change — ' || NEW.verb || ' ' || NEW.rel_path); END; CREATE TRIGGER IF NOT EXISTS campaign_activity_open AFTER INSERT ON deployments BEGIN diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 8d11972..398c29b 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -12,11 +12,11 @@ async function exclusive(id: number, action: () => Promise): Promise { import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; -import { dirname, join, posix } from 'node:path'; +import { join, posix } from 'node:path'; import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { sha256Buffer } from './files.js'; -import { ensureDest, getServer, heldByBeta } from './servers.js'; +import { getServer, heldByBeta } from './servers.js'; import { getBaselineForScope, getScript, @@ -30,7 +30,8 @@ import { type ScriptVersionRow } from './scripts.js'; import { composeServerCfg } from './config-compose.js'; -import { recordScript, removeScript, getServerScript } from './inventory.js'; +import { getServerScript, removeScript } from './inventory.js'; +import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js'; import { emit } from '../pipeline/bus.js'; import type { BetaStage } from '../pipeline/bus.js'; @@ -207,6 +208,19 @@ function nowIso(): string { return new Date().toISOString(); } +function unresolvedCampaignWrites(server: string): Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }> { + return db().prepare(`SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path + FROM staged_file_changes f + JOIN deployment_servers ds ON ds.deployment_id = f.source_id AND ds.server = f.server + LEFT JOIN server_deployments operation ON operation.id = ds.provisioning_id + WHERE f.source_kind = 'campaign' AND f.server = ? AND ds.server_deleted_at IS NULL + AND ds.restart_state IN ('pending','failed','held','manual') + AND (f.state = 'applying' OR (f.state = 'pending' + AND (ds.restart_state IN ('held','manual') OR operation.id IS NULL + OR operation.state IN ('queued','waiting','running','awaiting-health'))))`) + .all(server) as Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }>; +} + // ---------- Reads ---------- export function getBeta(id: number): BetaRow | undefined { @@ -370,6 +384,17 @@ export async function openBeta(input: OpenBetaInput): Promise { throw new BetaError('one or more servers are out of scope', 'out-of-scope', outOfScope); } + // A beta outranks campaigns during activation. Do not let it overtake older + // staged writes which would otherwise overwrite the tested version on closure. + // Legacy campaigns have no staged payload and retain their restart-only priority. + const queuedCampaigns = input.servers.flatMap(unresolvedCampaignWrites); + if (queuedCampaigns.length) { + const interrupted = queuedCampaigns.some(row => row.state === 'applying'); + throw new BetaError(interrupted + ? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.' + : 'Finish or resume queued file campaigns before opening a beta on these servers.', 'conflict', queuedCampaigns); + } + // Per-(server, script) conflict check. heldByBeta covers "another running // beta already claimed this target"; the previous stale-.OLD check is no // longer relevant now that snapshots live in the DB. @@ -383,14 +408,6 @@ export async function openBeta(input: OpenBetaInput): Promise { } if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts); - // Ensure destinations exist. The `ensureDest` helper covers only the - // script destination today; config/plugin destinations are created - // on-demand via mkdir during the write loop. - for (const name of input.servers) { - const ok = await ensureDest(name, true); - if (!ok.ok) throw new BetaError(`dest not usable for ${name}: ${ok.reason}`, 'dest-fail'); - } - // DB writes in one transaction. const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols const insertBeta = db().prepare( @@ -430,13 +447,12 @@ export async function openBeta(input: OpenBetaInput): Promise { db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map); } for (const r of resolved) { - const prior = getServerScript(name, r.script.target_rel_path); insertBetaServerScript.run( betaId, name, r.script.id, - prior ? 1 : 0, - prior?.script_version_id ?? null + 0, + null ); } } @@ -484,133 +500,61 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi await writeFile(join(archive, base), r.version.content); } - const insertSnapshot = db().prepare( - `INSERT INTO beta_file_snapshots (beta_id, server, rel_path, prior_content, prior_sha256, prior_size) - VALUES (?, ?, ?, ?, ?, ?)` - ); - - // On-disk deploy per server. If any write fails on a server, mark the - // whole server 'failed' — we can't restart a partially deployed server - // safely. Remaining artifacts on that server are skipped. - const updateStage = db().prepare( - `UPDATE beta_servers - SET stage = ?, error = ?, deployed_at = ?, updated_at = ? - WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL` - ); - + // Persist the requested bytes without touching any live destination. Provisioning + // captures the original file and inventory immediately before guarded activation. for (const name of names) { if (!isActiveBetaServer(betaId, name)) continue; - db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?") - .run(nowIso(), betaId, name); const serverRow = serverRows.find((r) => r.name === name)!; - let firstErr: string | null = null; - // 1. Non-collapsing artifacts: script/plugin — write directly to the - // target filename. Pre-beta bytes go into beta_file_snapshots so - // rollback can restore them. - for (const { script, version } of nonCollapsing) { - if (firstErr) break; - const dest = destOfKind(name, script.kind); - const targetPath = join(dest, script.target_rel_path); - try { - await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true }); - if (existsSync(targetPath)) { - const priorBytes = await readFile(targetPath); - if (!isActiveBetaServer(betaId, name)) break; - insertSnapshot.run( - betaId, - name, - script.target_rel_path, - priorBytes, - sha256Buffer(priorBytes), - priorBytes.byteLength - ); - db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id); - } - db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path); - if (!isActiveBetaServer(betaId, name)) break; - await writeFile(targetPath, version.content); - if (!isActiveBetaServer(betaId, name)) break; - recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`); - } catch (err) { - firstErr = `${script.name}: ${(err as Error).message}`; - } - } - // 2. Collapsing layer: compose the effective dedicated_sp.cfg from - // baseline + all applicable overlays, write in place. Pre-beta cfg - // bytes go into beta_file_snapshots keyed on the target rel_path. - const hasCollapsing = collapsingBaseline || collapsingOverlays.length > 0; - if (!firstErr && hasCollapsing) { - try { + try { + const changes: Parameters[0][] = nonCollapsing.map(({ script, version }) => ({ + sourceKind: 'beta' as const, sourceId: String(betaId), server: name, + kind: script.kind, relPath: script.target_rel_path, content: version.content, + scriptVersionId: version.id, inventorySource: `beta:${betaId}` + })); + if (collapsingBaseline || collapsingOverlays.length > 0) { const composedResult = composeCfgForServer(name, serverRow.game, serverRow.map, { inBetaBaseline: collapsingBaseline?.version, inBetaOverlays: collapsingOverlays }); if (!composedResult) { - throw new Error( - `no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays` - ); - } - const dest = destOfKind(name, 'config'); - await mkdir(dest, { recursive: true }); - const targetRel = configFileOf(name); - const targetPath = join(dest, targetRel); - if (existsSync(targetPath)) { - const priorBytes = await readFile(targetPath); - if (!isActiveBetaServer(betaId, name)) continue; - insertSnapshot.run( - betaId, - name, - targetRel, - priorBytes, - sha256Buffer(priorBytes), - priorBytes.byteLength - ); + throw new Error(`no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays`); } const composedBuf = Buffer.from(composedResult.text, 'utf8'); - db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel); - if (!isActiveBetaServer(betaId, name)) continue; - await writeFile(targetPath, composedBuf); - if (!isActiveBetaServer(betaId, name)) continue; - recordScript( - name, - targetRel, - null, - sha256Buffer(composedBuf), - composedBuf.byteLength, - `beta:${betaId}` - ); - // Archive the composed preview for operator visibility. + changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name, + kind: 'config', relPath: configFileOf(name), content: composedBuf, + scriptVersionId: null, inventorySource: `beta:${betaId}` }); await writeFile(join(archive, `${name}.${configFileOf(name)}`), composedBuf); - } catch (err) { - firstErr = `configs: ${(err as Error).message}`; } - } - const t = nowIso(); - if (!isActiveBetaServer(betaId, name)) continue; - if (firstErr) { - updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name); - emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` }); - } else { - updateStage.run('awaiting-empty', null, t, t, betaId, name); + if (!isActiveBetaServer(betaId, name)) continue; + db().transaction(() => { + for (const change of changes) queueFileChange(change); + db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") + .run(nowIso(), betaId, name); + })(); emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null }); + } catch (error) { + const message = `deploy: ${(error as Error).message}`; + db().prepare("UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") + .run(message, nowIso(), betaId, name); + emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: message }); } } - } // ---------- rollback ---------- export interface RollbackResult { betaId: number; + deploymentId: string | null; restored: string[]; dropped: string[]; warnings: string[]; } -export async function rollbackBeta(betaId: number): Promise { - return exclusive(betaId, () => rollbackBetaInternal(betaId)); +export async function rollbackBeta(betaId: number, options: { restart?: 'manual' | 'when-empty' } = {}): Promise { + return exclusive(betaId, () => rollbackBetaInternal(betaId, options)); } -async function rollbackBetaInternal(betaId: number): Promise { +async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise { const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'running' && beta.state !== 'ready-promote') { @@ -637,86 +581,77 @@ async function rollbackBetaInternal(betaId: number): Promise { WHERE beta_id = ? AND server = ? AND rel_path = ?` ); + const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = []; + const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'); + const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => { + const staged = isFileApplied('beta', String(betaId), bs.server, kind, path); + if (staged !== null) return staged; + // Existing betas predating staging wrote their files before deployed_at. + return !!changed.get(betaId, bs.server, path) || !!bs.deployed_at; + }; for (const bs of servers) { if (!isActiveBetaServer(betaId, bs.server)) { warnings.push(`${bs.server}: original server was deleted; retained history was skipped`); continue; } - if (!bs.deployed_at && bs.stage === 'queued') continue; for (const { script } of bundle) { - if (!isActiveBetaServer(betaId, bs.server)) break; - if (isCollapsingKind(script.kind)) continue; // handled once per server below - if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue; - const dest = destOfKind(bs.server, script.kind); - const targetPath = join(dest, script.target_rel_path); - + if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) continue; const bss = getBetaServerScript(betaId, bs.server, script.id); - if (bss?.had_original) { - const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as - | { prior_content: Buffer; prior_sha256: string; prior_size: number } - | undefined; - if (snap) { - await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true }); - if (!isActiveBetaServer(betaId, bs.server)) break; - await writeFile(targetPath, snap.prior_content); - if (!isActiveBetaServer(betaId, bs.server)) break; - if (bss.prior_version_id) { - recordScript(bs.server, script.target_rel_path, bss.prior_version_id, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`); - } else { - recordScript(bs.server, script.target_rel_path, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`); - } - restored.add(bs.server); - } else { - warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`); - } - } else { - // No prior file — the beta wrote a fresh file. Drop it. - if (existsSync(targetPath)) await rm(targetPath, { force: true }); - if (!isActiveBetaServer(betaId, bs.server)) break; - removeScript(bs.server, script.target_rel_path); - dropped.add(bs.server); + const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as + | { prior_content: Buffer; prior_sha256: string; prior_size: number } + | undefined; + if (bss?.had_original && !snap) { + warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`); + continue; } + restorations.push({ server: bs.server, kind: script.kind, relPath: script.target_rel_path, + content: snap?.prior_content ?? null, versionId: bss?.prior_version_id ?? null }); + (snap ? restored : dropped).add(bs.server); } - - // Collapsing config layer: single restore for the shared cfg file. - if (!isActiveBetaServer(betaId, bs.server)) continue; - if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) { - const dest = destOfKind(bs.server, 'config'); - const targetRel = configFileOf(bs.server); - const targetPath = join(dest, targetRel); + const targetRel = configFileOf(bs.server); + if (hasCollapsing && applied(bs, 'config', targetRel)) { const snap = getSnapshot.get(betaId, bs.server, targetRel) as | { prior_content: Buffer; prior_sha256: string; prior_size: number } | undefined; - if (snap) { - await mkdir(dest, { recursive: true }); - if (!isActiveBetaServer(betaId, bs.server)) continue; - await writeFile(targetPath, snap.prior_content); - if (!isActiveBetaServer(betaId, bs.server)) continue; - recordScript(bs.server, targetRel, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`); - restored.add(bs.server); - } else if (existsSync(targetPath)) { - // No snapshot ⇒ the beta wrote the cfg into a server that had - // no pre-existing file. Remove the beta-written cfg to leave - // the server clean. - await rm(targetPath, { force: true }); - if (!isActiveBetaServer(betaId, bs.server)) continue; - removeScript(bs.server, targetRel); - dropped.add(bs.server); - } + restorations.push({ server: bs.server, kind: 'config', relPath: targetRel, + content: snap?.prior_content ?? null, versionId: null }); + (snap ? restored : dropped).add(bs.server); } } - // Snapshots are single-use — safe to drop now that the disk is restored. - db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); + const deploymentId = restorations.length ? `dep-beta-rollback-${betaId}-${Date.now()}` : null; + db().transaction(() => { + cancelStagedFiles('beta', String(betaId)); + if (deploymentId) { + db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)") + .run(deploymentId, nowIso(), storedPlan(betaId) ? 'beta-test-plan' : 'ui', betaId); + for (const server of new Set(restorations.map(change => change.server))) { + db().prepare('INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)') + .run(deploymentId, server, restorations.filter(change => change.server === server).length, + options.restart === 'manual' ? 'held' : 'pending'); + } + for (const change of restorations) { + const file = db().prepare(`INSERT INTO deployment_files + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) + VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server, + change.content === null ? 'removed' : 'updated', change.relPath, + change.content === null ? null : sha256Buffer(change.content), change.content); + queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server, + kind: change.kind, relPath: change.relPath, content: change.content, + scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`, + deploymentFileId: Number(file.lastInsertRowid) }); + } + } + // The queued restoration owns a durable copy before snapshots are released. + db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); + db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId); + })(); + if (deploymentId) emit({ type: 'deployment-opened', deploymentId, action: 'rollback', servers: [...new Set(restorations.map(change => change.server))] }); // Wipe archive + close out beta. const archive = betaArchiveOf(betaId); if (existsSync(archive)) await rm(archive, { recursive: true, force: true }); - db() - .prepare(`UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?`) - .run(nowIso(), betaId); - // Per-version test-failed transition. Skip a version if (a) another beta // still gates it at ready-promote/promoted, or (b) it's the currently- // promoted version of its script (already-live code isn't "failed" by @@ -742,6 +677,7 @@ async function rollbackBetaInternal(betaId: number): Promise { return { betaId, + deploymentId, restored: [...restored].sort(), dropped: [...dropped].sort(), warnings @@ -915,8 +851,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom } } - // Write files. Track per-plan success and per-server first-error. - const planWritten = new Map(); + // Build durable activation work; live inventory changes only at activation. const serverErr = new Map(); // Collapsing config layer: one composed cfg per server. Union of beta @@ -928,6 +863,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom composedContent: Buffer; composedSha: string; hadFile: boolean; + requiresActivation: boolean; } const cfgPlans: CfgPlan[] = []; if (collapsing.length > 0) { @@ -951,7 +887,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom let priorContent: Buffer | null = null; let priorSha: string | null = null; let hadFile = false; - // Beta server: prior cfg bytes were snapshotted at openBeta. + // Beta server: preserve its original bytes captured at activation. if (betaServerSet.has(name)) { const snap = getSnapshotForPromote.get(betaId, name, targetRel) as | { prior_content: Buffer; prior_sha256: string } @@ -962,59 +898,32 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom hadFile = true; } } - if (!hadFile && existsSync(targetPath)) { - // Scope-expansion server (or beta server with no snapshot): - // current on-disk cfg is the "before". + if (!betaServerSet.has(name) && existsSync(targetPath)) { + // Capture an initial preview; activation records the actual before image. priorContent = await readFile(targetPath); priorSha = sha256Buffer(priorContent); hadFile = true; } - cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile }); + const requiresActivation = !betaServerSet.has(name) || !existsSync(targetPath) + || sha256Buffer(await readFile(targetPath)) !== composedSha; + cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation }); } } // Recheck after asynchronous snapshot reads, before any mutation. Execution uses // the captured target lists/content so later membership changes never expand it. checkReview(); - for (const p of plans) { - checkReview(); - const dest = destOfKind(p.server, p.script.kind); - try { - await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true }); - checkReview(); - await writeFile(join(dest, p.script.target_rel_path), p.version.content); - checkReview(); - planWritten.set(p, true); - } catch (err) { - if (err instanceof BetaError) throw err; - planWritten.set(p, false); - if (!serverErr.has(p.server)) serverErr.set(p.server, `${p.script.name}: ${(err as Error).message}`); - } - } - // Write composed cfg to each target. - const cfgWritten = new Map(); + // A scope expansion from another promotion may have been queued while this + // beta was active. Reassert the tested bytes after that older campaign, even + // when they already match the live file, so it cannot overwrite this release. + const queuedByServer = new Map(reviewed.preview.servers.map(server => [server.name, unresolvedCampaignWrites(server.name)])); + const plansToActivate = new Set(plans.filter(p => !betaServerSet.has(p.server) + || queuedByServer.get(p.server)?.some(row => row.kind === p.script.kind && row.rel_path === p.script.target_rel_path))); for (const cp of cfgPlans) { - checkReview(); - try { - const dest = destOfKind(cp.server, 'config'); - await mkdir(dest, { recursive: true }); - checkReview(); - const targetRel = configFileOf(cp.server); - const targetPath = join(dest, targetRel); - // Beta server already has composed content at targetPath; overwrite - // idempotently. Scope-expansion server: write in place. - await writeFile(targetPath, cp.composedContent); - checkReview(); - cfgWritten.set(cp, true); - } catch (err) { - if (err instanceof BetaError) throw err; - cfgWritten.set(cp, false); - const msg = `configs: ${(err as Error).message}`; - if (!serverErr.has(cp.server)) serverErr.set(cp.server, msg); - console.error(`[promote] ${cp.server}/configs failed:`, err); - } + if (queuedByServer.get(cp.server)?.some(row => row.kind === 'config' && row.rel_path === configFileOf(cp.server))) cp.requiresActivation = true; } - + const activationServers = new Set([...Array.from(plansToActivate, plan => plan.server), + ...cfgPlans.filter(plan => plan.requiresActivation).map(plan => plan.server)]); // Build the aggregate deployment. const depId = promoteDeploymentId(betaId); const startedAt = nowIso(); @@ -1049,13 +958,11 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom for (const { script, version } of bundle) { insertDepScript.run(depId, script.id, version.id, script.current_version_id, script.kind); } - // deployment_files — one row per successful (script, server) write for - // non-collapsing artifacts, plus one row per (server, dedicated_sp.cfg) - // for the composed config layer. + // Record the reviewed change. Scope expansion gets its actual before + // image when activated; tested servers keep their pre-beta history. for (const p of plans) { - if (!planWritten.get(p)) continue; const verb = p.hadFile ? 'updated' : 'added'; - insertDepFile.run( + const file = insertDepFile.run( depId, p.server, verb, @@ -1065,19 +972,16 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom p.priorContent, p.version.content ); - recordScript( - p.server, - p.script.target_rel_path, - p.version.id, - p.version.sha256, - p.version.size, - `promote:${depId}` - ); + if (plansToActivate.has(p)) { + queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server, + kind: p.script.kind, relPath: p.script.target_rel_path, content: p.version.content, + scriptVersionId: p.version.id, inventorySource: `promote:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) }); + } } for (const cp of cfgPlans) { - if (!cfgWritten.get(cp)) continue; const verb = cp.hadFile ? 'updated' : 'added'; - insertDepFile.run( + const file = insertDepFile.run( depId, cp.server, verb, @@ -1087,14 +991,12 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom cp.priorContent, cp.composedContent ); - recordScript( - cp.server, - configFileOf(cp.server), - null, - cp.composedSha, - cp.composedContent.byteLength, - `promote:${depId}` - ); + if (cp.requiresActivation) { + queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server, + kind: 'config', relPath: configFileOf(cp.server), content: cp.composedContent, + scriptVersionId: null, inventorySource: `promote:${depId}`, + deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(cp.server) }); + } } // deployment_servers — one row per unique server. Beta servers are // already restarted; scope-only servers still need the sweep to drain @@ -1106,9 +1008,9 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom continue; } const filesChanged = - plans.filter((p) => p.server === name && planWritten.get(p)).length + - cfgPlans.filter((c) => c.server === name && cfgWritten.get(c)).length; - if (betaServerSet.has(name)) { + plans.filter((p) => p.server === name).length + + cfgPlans.filter((c) => c.server === name).length; + if (!activationServers.has(name)) { const bsRestartedAt = db() .prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`) .get(betaId, name) as { restarted_at: string | null } | undefined; @@ -1124,16 +1026,15 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom })(); db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId); - // Snapshots are single-use — the promoted content is live on disk, so the - // pre-beta bytes are no longer needed. + // The deployment history owns the pre-beta bytes for future rollback. db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); const archive = betaArchiveOf(betaId); if (existsSync(archive)) await rm(archive, { recursive: true, force: true }); const landed = Array.from(new Set([ - ...plans.filter((p) => planWritten.get(p)).map((p) => p.server), - ...cfgPlans.filter((c) => cfgWritten.get(c)).map((c) => c.server) + ...plans.map((p) => p.server), + ...cfgPlans.map((c) => c.server) ])).sort(); emit({ type: 'beta-state', betaId, state: 'promoted' }); @@ -1144,7 +1045,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom deploymentId: depId, landed, scriptIds: bundle.map((b) => b.script.id), - awaitingActivation: reviewed.preview.servers.filter(server => !server.tested && !serverErr.has(server.name)).length, + awaitingActivation: [...activationServers].filter(name => !serverErr.has(name)).length, failedServers: [...serverErr.keys()] }; } diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index e7aa6ef..eaa7a17 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -1,21 +1,18 @@ -import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { readdir, readFile, stat } from 'node:fs/promises'; import { randomUUID } from 'node:crypto'; import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js'; -import { dirname, join, posix, relative, sep } from 'node:path'; +import { join, posix, relative, sep } from 'node:path'; import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { emit } from '../pipeline/bus.js'; import { sha256Buffer } from './files.js'; -import { - getServerScript, - recordScript, - removeScript -} from './inventory.js'; +import { getServerScript } from './inventory.js'; import { getBaselineForScope, getVersion } from './scripts.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; import { selectedZombieMap } from './map-aliases.js'; import { getServer, listServers } from './servers.js'; import { promotedArtifacts } from './promoted-artifacts.js'; +import { queueFileChange } from './staged-files.js'; // Destinations drift scans, in a stable order so the UI shows scripts first. // Baseline shares the config destination — one physical dir, two logical kinds. @@ -309,8 +306,8 @@ export async function previewDrift(servers?: string[]): Promise { export interface RunDriftResult { deploymentId: string | null; - // `removed` = untracked files deleted from disk (drift kind='untracked') - // `restored` = missing or existing artifacts installed and tracked + // `removed` and `restored` describe queued changes. Provisioning applies + // them only after the server passes its activation gates. perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[] }>; totalRemoved: number; totalRestored: number; @@ -381,9 +378,8 @@ export function createDriftRebuildCampaign( /** * Detect drift on the requested servers (or the whole fleet) and either - * report the plan (dryRun) or remove untracked files and install managed - * artifacts as a single deployment. The sweep restarts each affected server once - * it's empty and waits for it to come back healthy. + * report the plan (dryRun) or stage the changes as a single deployment. + * Provisioning applies them and restarts each affected server once it's empty. */ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { const { dryRun = false, triggeredBy = 'ui' } = input; @@ -471,15 +467,19 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { + const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null); + queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, + kind: entry.kind, relPath: entry.relPath, content: null, + inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), + expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) }); + })(); s.removed.push(entry.relPath); - insertFile.run(depId, s.server, 'removed', entry.relPath, entry.sha256, null, content, null); } else { - // Install the managed artifact without deleting the target first. + // Persist exact replacement bytes so waiting cannot change the plan. const written = await resolveManagedEntry(s.server, entry); if (!written) { s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`); @@ -493,20 +493,17 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { + const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath, + before ? sha256Buffer(before) : null, sha256Buffer(written), before, written); + queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, + kind: entry.kind, relPath: entry.relPath, content: written, + scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null, + inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), + expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null }); + })(); s.restored.push(entry.relPath); - insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath, - before ? sha256Buffer(before) : null, sha256Buffer(written), before, written); } } catch (err) { s.skipped.push(`${entry.relPath}: ${(err as Error).message}`); diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index d832aa6..67089cb 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -1,11 +1,8 @@ -import { mkdir, rm, writeFile } from 'node:fs/promises'; -import { existsSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { destOfKind, type ArtifactKind } from '../config.js'; +import type { ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { emit } from '../pipeline/bus.js'; -import { recordScript, removeScript } from './inventory.js'; import { getVersion } from './scripts.js'; +import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js'; // Roll a promoted deployment back to whatever versions were current before // the promote landed. Reads deployment_scripts as the source of truth for @@ -142,58 +139,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise(); + const staged = stagedFileChanges('campaign', promoteId); + // Composed baseline rows are physically staged as config files. Prefer the + // explicit link instead of inferring their destination from the library. + const kindForFile = (file: FileRow) => staged.find(row => row.deployment_file_id === file.id)?.kind ?? kindForRelPath(file.rel_path); for (const f of fileRows) { + // A promotion can still be waiting for empty servers. Withdrawing it + // must not restore or remove files that it never installed. Rows from + // older releases have no staging record and retain their audit meaning. + const kind = kindForFile(f); + const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind + && row.rel_path === f.rel_path && row.preserve_before_image); + if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue; const arr = perServer.get(f.server); if (arr) arr.push(f); else perServer.set(f.server, [f]); } const affectedServers: string[] = []; - const perServerFailures = new Map(); const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server); - for (const [server, files] of perServer) { - if (!active(server)) continue; - try { - for (const f of files) { - if (!active(server)) break; - const kind = kindForRelPath(f.rel_path); - const dest = destOfKind(server, kind); - const abs = join(dest, f.rel_path); - await mkdir(join(dest, dirname(f.rel_path)), { recursive: true }); - if (!active(server)) break; - const owningScript = scriptByTarget.get(f.rel_path); - if (f.verb === 'added') { - if (existsSync(abs)) await rm(abs, { force: true }); - if (!active(server)) break; - removeScript(server, f.rel_path); - } else if (f.verb === 'updated' && f.content_before) { - await writeFile(abs, f.content_before); - if (!active(server)) break; - const priorVer = owningScript?.prior_version_id - ? getVersion(owningScript.prior_version_id) - : null; - recordScript( - server, - f.rel_path, - priorVer?.id ?? null, - f.sha256_before!, - f.content_before.byteLength, - `rollback:${newDepId}` - ); - } else if (f.verb === 'removed' && f.content_before) { - await writeFile(abs, f.content_before); - if (!active(server)) break; - recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`); - } - } - if (active(server)) affectedServers.push(server); - } catch (err) { - perServerFailures.set(server, (err as Error).message); - } - } - // Write the new rollback deployment + mark the original. const insertDep = db().prepare( `INSERT INTO deployments @@ -216,6 +182,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise { + cancelStagedFiles('campaign', promoteId); + // Withdrawing this source leaves a shared restart available to other + // campaigns. Provisioning cancels an operation with no eligible sources. + db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL + WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId); insertDep.run(newDepId, startedAt); // Mirror the promote's script list — swapped: the "current" version // after rollback is the promote's prior; the "prior" of the rollback @@ -231,8 +202,8 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise file.verb !== 'added' && file.content_before === null)) { + insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely'); continue; } insertDepServer.run(newDepId, server, files.length, 'pending', null); @@ -241,7 +212,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise row.state === 'applied' || row.state === 'applying'); +} + +/** null identifies legacy work that wrote files before staging was introduced. */ +export function isFileApplied(sourceKind: SourceKind, sourceId: string, server: string, kind: ArtifactKind, relPath: string): boolean | null { + const row = stagedFileChanges(sourceKind, sourceId, server).find(row => row.kind === kind && row.rel_path === relPath); + return row ? row.state === 'applied' || row.state === 'applying' : null; +} + +function readExisting(path: string): Buffer | null { + try { return readFileSync(path); } + catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; } +} + +/** Only called after occupancy checks and reservation of the restart slot. + * Synchronous file replacement keeps cancellation, new requests, and deletion + * from interleaving between the final gates and dispatching the restart. + */ +export function applyStagedFilesForOperation(operationId: string, server: string): void { + const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s + ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server + WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`) + .all(operationId, server) as StagedFileChange[]; + if (!rows.length) return; + if (!statSync(serverDir(server)).isDirectory()) throw new Error('Server directory is unavailable'); + // Preflight every target before changing any files. An interrupted write + // needs inspection/rollback, never an automatic replay. + const plans = rows.map(row => { + if (row.state === 'applying') throw new Error('A previous file write was interrupted; inspect or roll back the campaign'); + const source = row.source_kind === 'campaign' + ? db().prepare(`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ? + AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`).get(row.source_id, server, operationId) + : db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ? + AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`).get(row.source_id, server, operationId); + if (!source) throw new Error('File change requester is no longer eligible'); + const path = targetPath(server, row.kind, row.rel_path); + const before = readExisting(path); + if (row.check_before && (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha) { + throw new Error(`${row.rel_path}: changed while queued; scan and review again`); + } + return { row, path, before }; + }); + for (const { row, path } of plans) { + // Earlier coalesced campaigns may have just updated the same path. + const before = readExisting(path); + const prior = getServerScript(server, row.rel_path); + db().transaction(() => { + if (row.source_kind === 'beta') { + if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots + (beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`) + .run(row.source_id, server, row.rel_path, before, sha256Buffer(before), before.length); + db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ? + AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`) + .run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, server, row.rel_path, row.kind); + db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)') + .run(row.source_id, server, row.rel_path); + } + if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET + verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`) + .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before, + before === null ? null : sha256Buffer(before), row.deployment_file_id); + db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id); + })(); + if (row.content === null) { + rmSync(path, { force: true }); + removeScript(server, row.rel_path); + } else { + mkdirSync(dirname(path), { recursive: true }); + // Preserve existing ownership, permissions, and symlink behavior. + // The durable applying checkpoint prevents replay after a partial write. + writeFileSync(path, row.content); + // A queued version can be deleted independently; the staged bytes + // still apply, but its optional inventory link must remain valid. + const version = row.script_version_id !== null && db().prepare('SELECT 1 FROM script_versions WHERE id = ?').get(row.script_version_id) + ? row.script_version_id : null; + recordScript(server, row.rel_path, version, sha256Buffer(row.content), row.content.length, row.inventory_source); + } + db().prepare("UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?") + .run(new Date().toISOString(), row.id); + if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') + .run(row.source_id, `${server}: applied ${row.rel_path}`); + } + for (const betaId of new Set(rows.filter(row => row.source_kind === 'beta').map(row => row.source_id))) { + db().prepare('UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL') + .run(new Date().toISOString(), new Date().toISOString(), betaId, server); + } +} diff --git a/src/lib/server/pipeline/beta-plan-runner.ts b/src/lib/server/pipeline/beta-plan-runner.ts index 26d5df9..4cd51a7 100644 --- a/src/lib/server/pipeline/beta-plan-runner.ts +++ b/src/lib/server/pipeline/beta-plan-runner.ts @@ -25,19 +25,10 @@ export function betaState(id: number, ready: boolean): void { } export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise { - const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server)); - const result = await rollbackBeta(id); - if (plan.rollbackRestart === 'when-empty' && servers.length) { - const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`; - db().transaction(() => { - db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)") - .run(deploymentId, new Date().toISOString(), id); - for (const server of servers) { - if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server); - } - })(); - activity(id, 'Restored files; rolling restarts will wait for empty servers'); - } else activity(id, 'Restored files; manual restarts are required to load them'); + const result = await rollbackBeta(id, { restart: plan.rollbackRestart }); + if (!result.deploymentId) activity(id, 'Cancelled queued changes; no deployed files need restoration'); + else if (plan.rollbackRestart === 'when-empty') activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers'); + else activity(id, 'File restoration is held; resume the rollback campaign to apply it with a safe restart'); if (result.warnings.length) activity(id, result.warnings.join('; ')); } diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 47d157a..10dd5e1 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -51,11 +51,17 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle } // Only not-yet-submitted campaign changes can share an activation. A later // file deployment must never be credited to an already running restart. - if (source.kind === 'campaign' && !row.restarted_at) { + // Restorations need their own activation. An older campaign can have + // stale file preconditions after a beta; its failure must not strand + // the rollback that would restore the server. + const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id); + if (source.kind === 'campaign' && !row.restarted_at && !rollback) { id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') + AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id + WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback') ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id; } if (!id) { diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index 295912e..fc204b5 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -3,6 +3,7 @@ import { db } from '../db/index.js'; import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js'; import { probeServerStatus } from '../fleet/status-evidence.js'; import { deploymentStage, finishDeployment } from './history.js'; +import { applyStagedFilesForOperation } from '../fleet/staged-files.js'; // Komodo queues restart work. Don't mistake the still-running old container // for a completed restart when an API kick immediately follows the request. @@ -64,16 +65,26 @@ export async function restartInRollout(server: string, options: { operationId: s if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted'; const reason = rolloutWaitReason(server, operationId); if (reason) return reason; - db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId); db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`) .run(server, new Date().toISOString(), operationId); - deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container'); return null; }).immediate(); if (waitReason) return { kind: 'queued', message: waitReason }; + // Every gate, including rollout capacity, has cleared. No async work may + // intervene between applying these files and submitting the restart. + try { + applyStagedFilesForOperation(operationId, server); + } catch (err) { + const message = `File activation failed: ${(err as Error).message}`; + db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); + finishDeployment(operationId, { ok: false, message }); + return { kind: 'failed', message }; + } + db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId); + deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container'); requestsInFlight.set(server, operationId); db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index 576bb3c..ea68768 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -29,12 +29,13 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' if (!maintenanceAllowed(incident.kind)) return 'held'; } let held = false; + let ready = false; for (const source of sources) { if (source.kind === 'campaign') { const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL') .get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined; if (row?.provisioning_id !== operation.id || !row.files_changed) continue; - if (['pending','failed'].includes(row.restart_state)) return 'ready'; + if (['pending','failed'].includes(row.restart_state)) ready = true; if (['held','manual'].includes(row.restart_state)) held = true; } else if (source.kind === 'beta') { const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id @@ -42,18 +43,18 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' .get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined; if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue; if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue; - if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready'; + if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true; if (row.outcome === 'paused') held = true; } else if (source.kind === 'maintenance') { const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined; if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) { if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held'; - return 'ready'; + ready = true; } - } else return 'ready'; + } else ready = true; } - return held ? 'held' : 'cancelled'; + return held ? 'held' : ready ? 'ready' : 'cancelled'; } function blocker(server: string, id: string): string | null { diff --git a/src/lib/ui/components/BetaPlanBuilder.svelte b/src/lib/ui/components/BetaPlanBuilder.svelte index 1db83c3..e146698 100644 --- a/src/lib/ui/components/BetaPlanBuilder.svelte +++ b/src/lib/ui/components/BetaPlanBuilder.svelte @@ -195,7 +195,7 @@
- +

Waiting for empty servers uses this window. Pausing and retrying do not extend it. Missing evidence at the deadline is inconclusive. Health checks continue until you manually promote.

diff --git a/src/lib/ui/status.ts b/src/lib/ui/status.ts index 695adad..138d938 100644 --- a/src/lib/ui/status.ts +++ b/src/lib/ui/status.ts @@ -11,7 +11,7 @@ const states: Record = { done: ['Completed', 'online'], passed: ['Test passed', 'online'], tested: ['Test passed', 'online'], promoted: ['Promoted', 'online'], 'rolled-back': ['Rolled back', 'neutral'], cancelled: ['Cancelled', 'neutral'], failed: ['Failed', 'danger'], 'test-failed': ['Test failed', 'danger'], interrupted: ['Interrupted', 'danger'], - warning: ['Completed with warnings', 'warning'], held: ['Held by beta', 'neutral'], + warning: ['Completed with warnings', 'warning'], held: ['On hold', 'neutral'], skipped: ['Withdrawn', 'neutral'], manual: ['Manual action needed', 'warning'], paused: ['Paused', 'warning'], inconclusive: ['Missing evidence', 'warning'], untested: ['Needs testing', 'neutral'], testing: ['Beta active', 'info'], empty: ['No versions', 'neutral'], attention: ['Needs attention', 'warning'] }; diff --git a/src/routes/api/betas/[id]/+server.ts b/src/routes/api/betas/[id]/+server.ts index 3a70f7e..3a5c2c8 100644 --- a/src/routes/api/betas/[id]/+server.ts +++ b/src/routes/api/betas/[id]/+server.ts @@ -33,7 +33,9 @@ export async function GET({ params }: { params: { id: string } }) { const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id); const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined; const promotion = campaign ? { ...campaign, - pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n, + pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled','skipped')").get(campaign.id) as { n: number }).n, failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null; - return json({ beta, scripts, servers, events, test: testView(id), promotion }); + const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1") + .get(id) as { id: string } | undefined; + return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null }); } diff --git a/src/routes/api/betas/[id]/retry/+server.ts b/src/routes/api/betas/[id]/retry/+server.ts index 267460d..212b5cc 100644 --- a/src/routes/api/betas/[id]/retry/+server.ts +++ b/src/routes/api/betas/[id]/retry/+server.ts @@ -3,6 +3,8 @@ import { db } from '$lib/server/db/index.js'; import { emit, type BetaStage } from '$lib/server/pipeline/bus.js'; import { storedPlan } from '$lib/server/fleet/beta-tests.js'; import { releaseRestartRequest } from '$lib/server/provisioning/index.js'; +import { deployBetaServers, getBeta, isActiveBetaServer } from '$lib/server/fleet/beta.js'; +import { stagedFileChanges } from '$lib/server/fleet/staged-files.js'; function parseId(raw: string): number | null { const n = Number.parseInt(raw, 10); @@ -14,18 +16,33 @@ function parseId(raw: string): number | null { export async function POST({ params, request }: { params: { id: string }; request: Request }) { const betaId = parseId(params.id); if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 }); + const beta = getBeta(betaId); + if (!beta || !['running', 'ready-promote'].includes(beta.state)) return json({ ok: false, error: 'Beta is closed or missing' }, { status: 409 }); if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 }); const body = (await request.json().catch(() => ({}))) as { server?: string }; if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 }); const row = db() - .prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?') + .prepare('SELECT stage, deployed_at, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?') .get(betaId, body.server) as - | { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null } + | { stage: string; deployed_at: string | null; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null } | undefined; if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 }); - if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 }); + if (!isActiveBetaServer(betaId, body.server)) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 }); if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 }); + if (stagedFileChanges('beta', String(betaId), body.server).some(change => change.state === 'applying')) { + return json({ ok: false, error: 'A file write was interrupted. Inspect the server and roll back before opening a fresh beta.' }, { status: 409 }); + } + + // Preparation may have failed before any bytes were queued (for example, + // an overlay without a baseline). Retrying must prepare its payload first. + if (!row.deployed_at && !row.restarted_at) { + try { await deployBetaServers(betaId, [body.server]); } + catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 409 }); } + const prepared = db().prepare('SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?') + .get(betaId, body.server) as { stage: string; error: string | null }; + if (prepared.stage === 'failed') return json({ ok: false, error: prepared.error }, { status: 409 }); + } // Best-effort resume point: if we already got past restart, go back to // health check; otherwise back to waiting for the server to be empty. diff --git a/src/routes/api/betas/[id]/test/+server.ts b/src/routes/api/betas/[id]/test/+server.ts index 3a254a4..8559aaf 100644 --- a/src/routes/api/betas/[id]/test/+server.ts +++ b/src/routes/api/betas/[id]/test/+server.ts @@ -1,7 +1,8 @@ import { json } from '@sveltejs/kit'; import { releaseRestartRequest } from '$lib/server/provisioning/index.js'; import { db } from '$lib/server/db/index.js'; -import { getBeta } from '$lib/server/fleet/beta.js'; +import { deployBetaServers, getBeta } from '$lib/server/fleet/beta.js'; +import { stagedFileChanges } from '$lib/server/fleet/staged-files.js'; import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js'; import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js'; import { parseBetaPlan } from '$lib/beta-plan.js'; @@ -52,7 +53,16 @@ export async function POST({ params, request }: { params: { id: string }; reques // Fresh attempts keep their predecessors as audit history. Preserve the original deadline: // operators open another beta if the overall testing window has expired. if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline'); - if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta'); + if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) { + throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta'); + } + const prepare = db().prepare("SELECT server FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL AND restarted_at IS NULL") + .all(id) as { server: string }[]; + if (prepare.length) { + await deployBetaServers(id, prepare.map(row => row.server)); + const failed = db().prepare("SELECT error FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL").get(id) as { error: string | null } | undefined; + if (failed) throw new Error(failed.error ?? 'Could not prepare beta files'); + } db().transaction(() => { const servers = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ?').all(id) as { server: string }[]; for (const { server } of servers) releaseRestartRequest(server, { kind: 'beta', id: String(id) }); diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 63bd69b..4962bff 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -17,7 +17,8 @@ export async function GET({ params }: { params: { id: string } }) { .prepare( `SELECT id, server, verb, rel_path, sha256_before, sha256_after, LENGTH(content_before) AS size_before, - LENGTH(content_after) AS size_after + LENGTH(content_after) AS size_after, + COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state FROM deployment_files WHERE deployment_id = ? ORDER BY server, rel_path` diff --git a/src/routes/api/campaigns/[id]/resume/+server.ts b/src/routes/api/campaigns/[id]/resume/+server.ts new file mode 100644 index 0000000..1a5c512 --- /dev/null +++ b/src/routes/api/campaigns/[id]/resume/+server.ts @@ -0,0 +1,25 @@ +import { json } from '@sveltejs/kit'; +import { db } from '$lib/server/db/index.js'; +import { releaseRestartRequest } from '$lib/server/provisioning/index.js'; +import { kick } from '$lib/server/pipeline/worker.js'; + +export function POST({ params }: { params: { id: string } }) { + const changed = db().transaction(() => { + const rows = db().prepare(`SELECT ds.server, d.state FROM deployment_servers ds + LEFT JOIN server_deployments d ON d.id = ds.provisioning_id + WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`) + .all(params.id) as { server: string; state: string | null }[]; + for (const row of rows) { + if (row.state && ['failed','interrupted','cancelled'].includes(row.state)) { + releaseRestartRequest(row.server, { kind: 'campaign', id: params.id }); + } + } + const result = db().prepare(`UPDATE deployment_servers SET restart_state = 'pending', error = NULL + WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`).run(params.id); + if (result.changes) db().prepare('UPDATE deployments SET finished_at = NULL WHERE id = ?').run(params.id); + return result.changes; + })(); + if (!changed) return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 }); + void kick().catch(err => console.error('[campaign] activation kick failed:', err)); + return json({ ok: true }, { status: 202 }); +} diff --git a/src/routes/betas/[id]/+page.svelte b/src/routes/betas/[id]/+page.svelte index c7240e8..db177e0 100644 --- a/src/routes/betas/[id]/+page.svelte +++ b/src/routes/betas/[id]/+page.svelte @@ -31,6 +31,7 @@ let servers = $state([]); let events = $state([]); let promotion = $state(null); + let rollbackCampaign = $state<{ id: string }|null>(null); let loading = $state(true), loadError = $state(''), flash = $state(''), success = $state(''); let acting = $state(null), request = 0; let reviewOpen = $state(false), preview = $state(null), reviewError = $state(''); @@ -44,7 +45,7 @@ const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length); const ranked = $derived([...servers].sort((a,b) => (a.stage === 'failed' ? 0 : a.stage === 'ready' ? 2 : 1) - (b.stage === 'failed' ? 0 : b.stage === 'ready' ? 2 : 1) || a.server.localeCompare(b.server))); const waiting = $derived(ranked.filter(server => server.stage !== 'ready')); - const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check activation on additional servers.' : 'Reuse these files and this plan when you are ready to test again.' + const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.' : hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.' : test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.' : test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.' @@ -60,6 +61,7 @@ const body = await res.json(); if (current !== request || requestedId !== id) return; test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null; + rollbackCampaign = body.rollback ?? null; now = Date.now(); loadError = ''; } catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; } finally { if (current === request && requestedId === id) loading = false; } @@ -86,20 +88,20 @@ throw new Error(body.error ?? 'Promotion could not be started.'); } reviewOpen = false; - success = `Promotion created. ${body.result.awaitingActivation ?? 0} additional servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`; + success = `Promotion created. ${body.result.awaitingActivation ?? 0} servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`; await load(); } catch (error) { reviewError = (error as Error).message; } finally { acting = null; } } async function rollback() { const reject = beta?.state === 'ready-promote'; - const restarts = test?.plan.rollbackRestart === 'manual' ? 'Restart servers manually to load the restored files.' : 'Restarts wait for empty servers.'; + const restarts = test?.plan.rollbackRestart === 'manual' ? 'Resume the rollback campaign when ready; files stay unchanged until safety checks pass.' : 'Files are restored just before restarting, after safety checks pass.'; if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.filter(server => !server.server_deleted_at).length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return; acting = 'rollback'; flash = ''; success = ''; try { const response = await fetch(`/api/betas/${id}/rollback`,{method:'POST'}), body = await response.json(); if (!response.ok || !body.ok) throw new Error(body.error ?? 'Rollback failed.'); - success = `Original files restored. ${restarts}`; await load(); + success = `Rollback queued. ${restarts}`; await load(); } catch(error) { flash = (error as Error).message; } finally { acting = null; } } @@ -142,6 +144,7 @@ {#if !closed}{/if} {#if closed}{/if} {#if promotion}{/if} + {#if rollbackCampaign}{/if} {#if promotion}

{promotion.pending} servers awaiting activation · {promotion.failed} need attention. Promoting a version does not mean it is already running everywhere.

{/if} {#if flash}{/if} @@ -166,7 +169,7 @@ {#if reviewError}{/if} {#if preview}

{preview.servers.filter(server => server.tested).length} tested servers and {preview.servers.filter(server => !server.tested).length} additional eligible servers.

-

Files are installed when you confirm. Tested servers have already restarted. Additional servers restart when empty, then their health is checked. Eligibility is determined by each file's current scope.

+

Confirming queues the file changes. Each server waits until empty and restart capacity is available, then installs its files and restarts. Tested servers only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.

Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed settings change, you must review again.

{#each preview.blockers as blocker}{/each}
{#each preview.files as file}
{file.name} · {artifactVersionLabel(file)} · {file.servers.length} servers

{file.path} · {file.scope} · SHA {file.sha256.slice(0,12)}

    {#each file.servers as name}
  • {preview.servers.find(server => server.name === name)?.tested ? 'Already tested' : 'Additional target · restart when empty'}
    • {/each}
{/each}
diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 053e3dd..49c0191 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -20,6 +20,7 @@ id: number; server: string; verb: string; rel_path: string; sha256_before: string | null; sha256_after: string | null; size_before: number | null; size_after: number | null; + apply_state: 'pending' | 'applying' | 'applied' | 'cancelled'; } interface ServerRow { deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number; @@ -40,6 +41,8 @@ let detail = $state(null); let loadError = $state(''); + let resuming = $state(false); + let resumeError = $state(''); let request = 0; let diffOpen = $state(false); let diffError = $state(''); @@ -48,7 +51,8 @@ let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null); let retryFile = $state(null); const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0); - const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0); + const withdrawn = $derived(detail?.servers.filter(server => ['skipped', 'cancelled'].includes(server.restart_state)).length ?? 0); + const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && !['done', 'skipped', 'cancelled'].includes(server.restart_state)).length ?? 0); const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0); const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => { const rank: Record = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 }; @@ -73,6 +77,16 @@ onMount(() => pollVisible(loadDetail, 5000, false)); + async function resume() { + resuming = true; resumeError = ''; + try { + const response = await fetch(`/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, { method: 'POST' }); + if (!response.ok) throw new Error((await response.json()).error ?? 'Could not resume campaign.'); + await loadDetail(); + } catch (error) { resumeError = (error as Error).message; } + finally { resuming = false; } + } + async function loadDiff(f: FileRow) { retryFile = f; const id = $page.params.id; @@ -123,10 +137,21 @@ {/snippet} -

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}

{#if unresolved || deletedIncomplete}

Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}

{:else}

Every recorded server completed activation.

{/if}
+
+

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}

+

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}

+ {#if unresolved || deletedIncomplete}

Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}

+ {:else if withdrawn}

Withdrawn changes will not be activated by this campaign.

+ {:else}

Every recorded server completed activation.

{/if} +
{#if detail.watchdogIncidentId}

Started by Maintenance · View incident and recovery history

{/if} + {#if detail.servers.some(server => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))} +

Apply held changes and restart after the safety checks pass.

+ {/if} + {#if resumeError}{/if} +
Started{fmt(detail.deployment.started_at)}
@@ -182,6 +207,7 @@ {#each files as f (f.id)}