From 9fd448339d62d89a8d68decbc8527e88c0a44b23 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 00:16:43 -0400 Subject: [PATCH] Defer promotion snapshots to campaign activation --- docs/beta-test-plans.md | 12 +++ src/lib/operation-flow.ts | 17 +++- src/lib/server/db/index.ts | 3 + src/lib/server/db/schema.sql | 4 +- src/lib/server/fleet/beta.ts | 37 ++----- src/lib/server/fleet/rollback.ts | 8 +- src/lib/server/fleet/staged-files.ts | 12 ++- src/lib/server/provisioning/native.ts | 8 +- src/routes/api/campaigns/[id]/+server.ts | 2 +- .../campaigns/[id]/file/[fileId]/+server.ts | 4 +- src/routes/betas/[id]/+page.svelte | 2 +- src/routes/campaigns/[id]/+page.svelte | 12 ++- tests/unit/beta-promotion.test.ts | 99 ++++++++++++++++++- tests/unit/native-provisioning.test.ts | 6 +- tests/unit/operation-history.test.ts | 9 +- 15 files changed, 179 insertions(+), 56 deletions(-) diff --git a/docs/beta-test-plans.md b/docs/beta-test-plans.md index cb2cd91..f55aa97 100644 --- a/docs/beta-test-plans.md +++ b/docs/beta-test-plans.md @@ -207,6 +207,18 @@ Restoration and restart both wait for an empty server and restart capacity. The manual option holds those changes in a rollback campaign until you select **Resume when empty**. Promotion is always manual. +Confirming promotion creates a campaign without reading the existing files on +additional servers. Each server waits for an empty lobby and restart capacity, +then captures rollback snapshots immediately before applying its queued files +and restarting. Campaign history records snapshot capture, and file comparisons +become available once those snapshots are confirmed. Tested servers retain their +original pre-beta snapshots; their composed configuration is still checked at +promotion to determine whether another activation is needed. + +Withdrawing changes that have not activated needs no snapshot and leaves those +servers' files untouched. If activation was submitted but its snapshot has not +been confirmed, reconcile the provisioning operation before rolling back. + Pause holds evaluation and wave advancement. Resume retains completed progress and the current revision's deadline, but starts new timing intervals. A fresh attempt resets progress and restarts deployed servers when empty. Recovery cycles repeat diff --git a/src/lib/operation-flow.ts b/src/lib/operation-flow.ts index 693b64e..5e0da1a 100644 --- a/src/lib/operation-flow.ts +++ b/src/lib/operation-flow.ts @@ -29,12 +29,19 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo : 'pending' })); } -export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] { +export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean, + files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] { const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state)); - const positions = active.map(s => s.restart_state === 'done' ? 4 - : s.restarted_at ? 3 : 0); - const current = positions.length ? Math.min(...positions) : finished ? 4 : 0; - return ['Wait for empty servers and restart capacity', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({ + const positions = active.map(s => { + if (s.restart_state === 'done') return 5; + if (s.restarted_at) return 4; + const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled'); + if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2; + if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3; + return 0; + }); + const current = positions.length ? Math.min(...positions) : finished ? 5 : 0; + return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({ label, state: i < current ? 'done' : i !== current ? 'pending' : active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed' : active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current' diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 7090a0a..90e66d7 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -317,6 +317,9 @@ export function db(): Database.Database { handle.exec(schemaSql); migrateProvisioningActions(handle); migrateWatchdogConstraints(handle); + migrateTable(handle, 'deployment_files', [ + { name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' } + ]); migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]); migrateTable(handle, 'server_deployments', [ { name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' }, diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index dd955bf..76de4c5 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -365,7 +365,9 @@ CREATE TABLE IF NOT EXISTS deployment_files ( sha256_before TEXT, sha256_after TEXT, content_before BLOB, - content_after BLOB + content_after BLOB, + -- An unknown before image is distinct from a confirmed absent file. + snapshot_pending INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_deployment_files_dep ON deployment_files(deployment_id); diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 506e0b0..a44374e 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -35,7 +35,7 @@ import { type ScriptVersionRow } from './scripts.js'; import { composeServerCfg } from './config-compose.js'; -import { getServerScript, removeScript } from './inventory.js'; +import { removeScript } from './inventory.js'; import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js'; import { emit } from '../pipeline/bus.js'; import { cancelProvisioningJob } from '../provisioning/cancellation.js'; @@ -976,7 +976,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers; for (const name of targets) { - const dest = destOfKind(name, script.kind); let plan: Plan; if (betaServerSet.has(name)) { const bss = getBetaServerScript(betaId, name, script.id); @@ -1001,21 +1000,9 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false }; } } else { - // Non-beta matching-scope server: current on-disk file (if any) - // is the "before" state for a promote-then-rollback round trip. - const targetPath = join(dest, script.target_rel_path); - if (await exists(targetPath)) { - const priorContent = await readFile(targetPath); - const inv = getServerScript(name, script.target_rel_path); - plan = { - script, version, server: name, - priorVersionId: inv?.script_version_id ?? null, - priorSha: sha256Buffer(priorContent), - priorContent, hadFile: true - }; - } else { - plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false }; - } + // Campaign activation captures the actual before image just + // before replacing this file. Do not wait on remote reads here. + plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false }; } plans.push(plan); } @@ -1068,12 +1055,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom hadFile = true; } } - if (!betaServerSet.has(name) && await exists(targetPath)) { - // Capture an initial preview; activation records the actual before image. - priorContent = await readFile(targetPath); - priorSha = sha256Buffer(priorContent); - hadFile = true; - } const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath) || sha256Buffer(await readFile(targetPath)) !== composedSha; cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation }); @@ -1114,8 +1095,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom ); const insertDepFile = db().prepare( `INSERT INTO deployment_files - (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` ); const uniqueServers = Array.from(new Set([ @@ -1140,7 +1121,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom p.priorSha, p.version.sha256, p.priorContent, - p.version.content + p.version.content, + betaServerSet.has(p.server) ? 0 : 1 ); if (plansToActivate.has(p)) { queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server, @@ -1159,7 +1141,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom cp.priorSha, cp.composedSha, cp.priorContent, - cp.composedContent + cp.composedContent, + betaServerSet.has(cp.server) ? 0 : 1 ); if (cp.requiresActivation) { queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server, diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index b0e9a13..b6cbcfd 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -11,7 +11,7 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } // pre-multi-script legacy rows. export class RollbackError extends Error { - constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back') { + constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') { super(message); } } @@ -42,6 +42,7 @@ interface FileRow { sha256_after: string | null; content_before: Buffer | null; content_after: Buffer | null; + snapshot_pending: number; } interface ScriptTargetRow { @@ -100,7 +101,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise row.server === f.server && row.kind === kind && row.rel_path === f.rel_path && row.preserve_before_image); if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue; + if (f.snapshot_pending) { + throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending'); + } const arr = perServer.get(f.server); if (arr) arr.push(f); else perServer.set(f.server, [f]); diff --git a/src/lib/server/fleet/staged-files.ts b/src/lib/server/fleet/staged-files.ts index 1beb4c0..e3ce1c1 100644 --- a/src/lib/server/fleet/staged-files.ts +++ b/src/lib/server/fleet/staged-files.ts @@ -130,10 +130,14 @@ export function applyStagedFilesForOperation(operationId: string, server: string if (row.source_kind === 'beta') { captureBetaBeforeImage(row, before); } - if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET - verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`) - .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before, - before === null ? null : sha256Buffer(before), row.deployment_file_id); + if (row.deployment_file_id && !row.preserve_before_image) { + db().prepare(`UPDATE deployment_files SET + verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`) + .run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before, + before === null ? null : sha256Buffer(before), row.deployment_file_id); + if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') + .run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`); + } db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id); })(); if (row.content === null) { diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 781d8fb..1573593 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -240,8 +240,12 @@ function applyReceipt(job: Job, result: Record) { captureBetaBeforeImage(row, bytes); db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); } - if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?') - .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); + if (row.deployment_file_id && !row.preserve_before_image) { + db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?') + .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); + if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') + .run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`); + } if (row.content === null) removeScript(job.server, row.rel_path); else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null, sha256Buffer(row.content), row.content.length, row.inventory_source); diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 137be9f..1647488 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -15,7 +15,7 @@ export async function GET({ params }: { params: { id: string } }) { // it opens the diff viewer. const files = db() .prepare( - `SELECT id, server, verb, rel_path, sha256_before, sha256_after, + `SELECT id, server, verb, rel_path, sha256_before, sha256_after, snapshot_pending, LENGTH(content_before) AS size_before, LENGTH(content_after) AS size_after, COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state diff --git a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts index 4b6b2b5..a858f3a 100644 --- a/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts +++ b/src/routes/api/campaigns/[id]/file/[fileId]/+server.ts @@ -10,7 +10,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } } } const row = db() .prepare( - `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after + `SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending FROM deployment_files WHERE deployment_id = ? AND id = ?` ) @@ -22,9 +22,11 @@ export async function GET({ params }: { params: { id: string; fileId: string } } sha256_after: string | null; content_before: Buffer | null; content_after: Buffer | null; + snapshot_pending: number; } | undefined; if (!row) throw error(404, 'file not found'); + if (row.snapshot_pending) return json({ error: 'The rollback snapshot is not available yet. It is captured immediately before file activation.' }, { status: 409 }); return json({ verb: row.verb, diff --git a/src/routes/betas/[id]/+page.svelte b/src/routes/betas/[id]/+page.svelte index 4246cd5..a3c44d4 100644 --- a/src/routes/betas/[id]/+page.svelte +++ b/src/routes/betas/[id]/+page.svelte @@ -191,7 +191,7 @@ {#if reviewError}{/if} {#if preview}

{preview.servers.filter(server => server.tested).length} tested servers and {preview.servers.filter(server => !server.tested).length} additional eligible servers.

-

Confirming queues the file changes. Each server waits until empty and restart capacity is available, then installs its files and restarts. Tested servers only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.

+

Confirming creates a campaign and queues the file changes. Each server waits until empty and restart capacity is available, then captures rollback snapshots, installs its files, and restarts. Tested servers keep their pre-beta snapshots and only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.

Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed settings change, you must review again.

{#each preview.blockers as blocker}{/each}
{#each preview.files as file}
{file.name} · {artifactVersionLabel(file)} · {file.servers.length} servers

{file.path} · {file.scope} · SHA {file.sha256.slice(0,12)}

    {#each file.servers as name}
  • {preview.servers.find(server => server.name === name)?.tested ? 'Already tested' : 'Additional target · restart when empty'}
    • {/each}
{/each}
diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index da0ba30..a3a60ae 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -23,6 +23,7 @@ sha256_before: string | null; sha256_after: string | null; size_before: number | null; size_after: number | null; apply_state: 'pending' | 'applying' | 'applied' | 'cancelled'; + snapshot_pending: number; } interface ServerRow { deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number; @@ -97,7 +98,7 @@ diffOpen = true; diffLoading = true; diffError = ''; openFile = null; try { const response = await fetch('/api/campaigns/' + id + '/file/' + f.id); - if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : 'Could not load this file comparison. Close and select it again to retry.'); + if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : response.status === 409 ? 'The rollback snapshot is not available yet. Refresh after activation to compare files.' : 'Could not load this file comparison. Close and select it again to retry.'); const body = await response.json(); if (id === $page.params.id && current === diffRequest) openFile = {id:f.id, ...body}; } catch (error) { if (id === $page.params.id && current === diffRequest) diffError = (error as Error).message; } @@ -144,7 +145,7 @@

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}

-

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}

+

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. The campaign captures rollback snapshots immediately before applying files, then restarts the server and verifies health. Tested servers keep their original pre-beta snapshots.'}

{#if unresolved || deletedIncomplete}

Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}

{:else if withdrawn}

Withdrawn changes will not be activated by this campaign.

{:else}

Every recorded server completed activation.

{/if} @@ -211,8 +212,9 @@

server.server === srv)?.server_deleted_at} />

{#each files as f (f.id)} -
- !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'))} /> + !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'))} /> {#if diffError} {:else if diffLoading} diff --git a/tests/unit/beta-promotion.test.ts b/tests/unit/beta-promotion.test.ts index b7a37ee..9035da7 100644 --- a/tests/unit/beta-promotion.test.ts +++ b/tests/unit/beta-promotion.test.ts @@ -1,7 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import Database from 'better-sqlite3'; import { existsSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { activateFiles } from '../helpers/staged-files.js'; @@ -13,7 +14,7 @@ beforeEach(async () => { const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'map' = 'game') { const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); @@ -31,6 +32,92 @@ async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'ma const destination = () => join(fixture.t5, servers[1], 'runtime/plutonium/storage/t5/raw/scripts/sp/review.gsc'); describe('reviewed beta promotion', () => { + it('upgrades existing campaign history without marking known snapshots pending', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy', 'promote', ?)").run(new Date().toISOString()); + db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before) VALUES ('legacy', ?, 'updated', 'old.gsc', ?)") + .run(servers[0], Buffer.from('original')); + closeDb(); + const legacy = new Database(fixture.dbPath); + try { legacy.exec('ALTER TABLE deployment_files DROP COLUMN snapshot_pending'); } + finally { legacy.close(); } + expect(db().prepare('SELECT snapshot_pending, content_before FROM deployment_files').get()) + .toEqual({ snapshot_pending: 0, content_before: Buffer.from('original') }); + }); + it.each(['script', 'baseline'] as const)('queues %s promotion without contacting additional host agents', async kind => { + const { betaId } = await ready(kind); + const { prepareNativeHost } = await import('../helpers/native-host.js'); + await prepareNativeHost([servers[1]]); + const native = await import('../../src/lib/server/backends/native.js'); + const request = vi.spyOn(native, 'nativeRequest').mockRejectedValue(new Error('Agent unavailable')); + const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); + const result = await promoteBeta(betaId, previewBetaPromotion(betaId).token); + expect(result.awaitingActivation).toBe(1); + expect(request).not.toHaveBeenCalled(); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(db().prepare('SELECT snapshot_pending FROM deployment_files WHERE deployment_id = ? AND server = ?') + .get(result.deploymentId, servers[1])).toEqual({ snapshot_pending: 1 }); + }); + + it.each([ + { kind: 'script', before: 'changed while queued' }, + { kind: 'baseline', before: 'configuration changed while queued' }, + { kind: 'script', before: '' }, + { kind: 'script', before: null } + ] as const)('captures activation-time bytes and restores them: %j', async ({ kind, before }) => { + const { betaId } = await ready(kind); + const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js'); + const path = kind === 'script' ? destination() : join(destOfKind(servers[1], 'config'), configFileOf(servers[1])); + const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); + const result = await promoteBeta(betaId); + const { db } = await import('../../src/lib/server/db/index.js'); + const file = db().prepare('SELECT id FROM deployment_files WHERE deployment_id = ? AND server = ?') + .get(result.deploymentId, servers[1]) as { id: number }; + const { GET: diff } = await import('../../src/routes/api/campaigns/[id]/file/[fileId]/+server.js'); + const params = { id: result.deploymentId, fileId: String(file.id) }; + expect((await diff({ params })).status).toBe(409); + // An edit after confirmation must be the rollback target, not an older preview. + if (before !== null) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, before); + } + await activateFiles('campaign', result.deploymentId); + expect(db().prepare('SELECT snapshot_pending, verb, content_before FROM deployment_files WHERE id = ?').get(file.id)) + .toEqual({ snapshot_pending: 0, verb: before === null ? 'added' : 'updated', content_before: before === null ? null : Buffer.from(before) }); + expect(db().prepare('SELECT message FROM campaign_events WHERE deployment_id = ?').all(result.deploymentId)) + .toContainEqual({ message: `${servers[1]}: captured rollback snapshot for ${kind === 'script' ? 'review.gsc' : configFileOf(servers[1])}` }); + expect((await (await diff({ params })).json()).before).toBe(before); + const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const rollback = await rollbackPromoteDeployment(result.deploymentId); + await activateFiles('campaign', rollback.deploymentId); + if (before === null) expect(existsSync(path)).toBe(false); + else expect(await readFile(path, 'utf8')).toBe(before); + }); + + it('withdraws unactivated changes without needing snapshots, preserving existing files', async () => { + const { betaId } = await ready(); + await writeFile(destination(), 'untouched'); + const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); + const result = await promoteBeta(betaId); + const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + const rollback = await rollbackPromoteDeployment(result.deploymentId); + expect(rollback.affectedServers).not.toContain(servers[1]); + await activateFiles('campaign', rollback.deploymentId); + expect(await readFile(destination(), 'utf8')).toBe('untouched'); + }); + + it('blocks rollback without mutation when activation has not confirmed its snapshot', async () => { + const { betaId } = await ready(); + const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); + const result = await promoteBeta(betaId); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE source_id = ?").run(result.deploymentId); + const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js'); + await expect(rollbackPromoteDeployment(result.deploymentId)).rejects.toMatchObject({ kind: 'snapshot-pending' }); + expect(db().prepare('SELECT rolled_back_by FROM deployments WHERE id = ?').get(result.deploymentId)).toEqual({ rolled_back_by: null }); + expect(db().prepare("SELECT COUNT(*) n FROM deployments WHERE action = 'rollback'").get()).toEqual({ n: 0 }); + }); + it('previews exact versions and additional targets without writing, then links the activation campaign', async () => { const { betaId, script, version } = await ready(); const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js'); @@ -42,6 +129,12 @@ describe('reviewed beta promotion', () => { const result = await promoteBeta(betaId, preview.token); expect(result).toMatchObject({ awaitingActivation: 1, failedServers: [] }); expect(result.deploymentId).toContain('promote'); + const { GET: campaign } = await import('../../src/routes/api/campaigns/[id]/+server.js'); + const summary = await (await campaign({ params: { id: result.deploymentId } })).json(); + expect(summary.files).toEqual(expect.arrayContaining([ + expect.objectContaining({ server: servers[0], snapshot_pending: 0 }), + expect.objectContaining({ server: servers[1], snapshot_pending: 1 }) + ])); expect(existsSync(destination())).toBe(false); await activateFiles('campaign', result.deploymentId); expect(await readFile(destination(), 'utf8')).toBe('main(){}'); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index 058c059..5130088 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -697,7 +697,7 @@ describe('native instance provisioning', () => { const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes'); db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString()); db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name); - const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after) VALUES('campaign',?,'updated','hud.gsc',?)").run(job.name, after).lastInsertRowid); + const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, after).lastInsertRowid); const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); @@ -714,13 +714,13 @@ describe('native instance provisioning', () => { remote.players = 0; await advance(); const activation = (await operationFor(restart.id))!; expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } }); - expect(db().prepare('SELECT content_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null }); + expect(db().prepare('SELECT content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 }); const files = stage.payload.files as Array<{ id: number; path: string }>; reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } }); await advance(); expect((await detail(restart.id)).state).toBe('done'); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied'); - expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex') }); + expect(db().prepare('SELECT content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex')); }); diff --git a/tests/unit/operation-history.test.ts b/tests/unit/operation-history.test.ts index 4aef20e..025830f 100644 --- a/tests/unit/operation-history.test.ts +++ b/tests/unit/operation-history.test.ts @@ -59,8 +59,15 @@ describe('operation progress', () => { expect(campaignFlow([{ restart_state: 'failed', files_changed: 0, restarted_at: null }], false)[0].state).toBe('failed'); }); it('requires health verification after campaign restart and preserves manual work', () => { - expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'current']); + expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'done', 'current']); expect(campaignFlow([{ restart_state: 'manual', files_changed: 1, restarted_at: null }], true).some(s => s.state === 'current')).toBe(false); expect(campaignFlow([{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], true).every(s => s.state === 'done')).toBe(true); }); + it('shows snapshot confirmation before file application and restart', () => { + const servers = [{ server: 'test', restart_state: 'pending', files_changed: 1, restarted_at: null }]; + const files = [{ server: 'test', snapshot_pending: 1, apply_state: 'applying' }]; + expect(campaignFlow(servers, false, files)[1]).toEqual({ label: 'Capture rollback snapshots', state: 'current' }); + expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0 }])[2].state).toBe('current'); + expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0, apply_state: 'applied' }])[3].state).toBe('current'); + }); });