Defer promotion snapshots to campaign activation
This commit is contained in:
@@ -207,6 +207,18 @@ Restoration and restart both wait for an empty server and restart capacity.
|
||||
The manual option holds those changes in a rollback campaign until you select
|
||||
**Resume when empty**. Promotion is always manual.
|
||||
|
||||
Confirming promotion creates a campaign without reading the existing files on
|
||||
additional servers. Each server waits for an empty lobby and restart capacity,
|
||||
then captures rollback snapshots immediately before applying its queued files
|
||||
and restarting. Campaign history records snapshot capture, and file comparisons
|
||||
become available once those snapshots are confirmed. Tested servers retain their
|
||||
original pre-beta snapshots; their composed configuration is still checked at
|
||||
promotion to determine whether another activation is needed.
|
||||
|
||||
Withdrawing changes that have not activated needs no snapshot and leaves those
|
||||
servers' files untouched. If activation was submitted but its snapshot has not
|
||||
been confirmed, reconcile the provisioning operation before rolling back.
|
||||
|
||||
Pause holds evaluation and wave advancement. Resume retains completed progress and the
|
||||
current revision's deadline, but starts new timing intervals. A fresh attempt resets
|
||||
progress and restarts deployed servers when empty. Recovery cycles repeat
|
||||
|
||||
@@ -29,12 +29,19 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo
|
||||
: 'pending' }));
|
||||
}
|
||||
|
||||
export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] {
|
||||
export function campaignFlow(servers: { server?: string; restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean,
|
||||
files: { server: string; snapshot_pending: number; apply_state: string }[] = []): OperationStep[] {
|
||||
const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state));
|
||||
const positions = active.map(s => s.restart_state === 'done' ? 4
|
||||
: s.restarted_at ? 3 : 0);
|
||||
const current = positions.length ? Math.min(...positions) : finished ? 4 : 0;
|
||||
return ['Wait for empty servers and restart capacity', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
|
||||
const positions = active.map(s => {
|
||||
if (s.restart_state === 'done') return 5;
|
||||
if (s.restarted_at) return 4;
|
||||
const changes = files.filter(f => f.server === s.server && f.apply_state !== 'cancelled');
|
||||
if (changes.some(f => f.apply_state === 'applying')) return changes.some(f => f.snapshot_pending) ? 1 : 2;
|
||||
if (changes.length && changes.every(f => f.apply_state === 'applied')) return 3;
|
||||
return 0;
|
||||
});
|
||||
const current = positions.length ? Math.min(...positions) : finished ? 5 : 0;
|
||||
return ['Wait for empty servers and restart capacity', 'Capture rollback snapshots', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
|
||||
label, state: i < current ? 'done' : i !== current ? 'pending'
|
||||
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
|
||||
: active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
|
||||
|
||||
@@ -317,6 +317,9 @@ export function db(): Database.Database {
|
||||
handle.exec(schemaSql);
|
||||
migrateProvisioningActions(handle);
|
||||
migrateWatchdogConstraints(handle);
|
||||
migrateTable(handle, 'deployment_files', [
|
||||
{ name: 'snapshot_pending', ddl: 'ALTER TABLE deployment_files ADD COLUMN snapshot_pending INTEGER NOT NULL DEFAULT 0' }
|
||||
]);
|
||||
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
|
||||
migrateTable(handle, 'server_deployments', [
|
||||
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
|
||||
|
||||
@@ -365,7 +365,9 @@ CREATE TABLE IF NOT EXISTS deployment_files (
|
||||
sha256_before TEXT,
|
||||
sha256_after TEXT,
|
||||
content_before BLOB,
|
||||
content_after BLOB
|
||||
content_after BLOB,
|
||||
-- An unknown before image is distinct from a confirmed absent file.
|
||||
snapshot_pending INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deployment_files_dep ON deployment_files(deployment_id);
|
||||
|
||||
@@ -35,7 +35,7 @@ import {
|
||||
type ScriptVersionRow
|
||||
} from './scripts.js';
|
||||
import { composeServerCfg } from './config-compose.js';
|
||||
import { getServerScript, removeScript } from './inventory.js';
|
||||
import { removeScript } from './inventory.js';
|
||||
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
|
||||
import { emit } from '../pipeline/bus.js';
|
||||
import { cancelProvisioningJob } from '../provisioning/cancellation.js';
|
||||
@@ -976,7 +976,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers;
|
||||
|
||||
for (const name of targets) {
|
||||
const dest = destOfKind(name, script.kind);
|
||||
let plan: Plan;
|
||||
if (betaServerSet.has(name)) {
|
||||
const bss = getBetaServerScript(betaId, name, script.id);
|
||||
@@ -1001,21 +1000,9 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
|
||||
}
|
||||
} else {
|
||||
// Non-beta matching-scope server: current on-disk file (if any)
|
||||
// is the "before" state for a promote-then-rollback round trip.
|
||||
const targetPath = join(dest, script.target_rel_path);
|
||||
if (await exists(targetPath)) {
|
||||
const priorContent = await readFile(targetPath);
|
||||
const inv = getServerScript(name, script.target_rel_path);
|
||||
plan = {
|
||||
script, version, server: name,
|
||||
priorVersionId: inv?.script_version_id ?? null,
|
||||
priorSha: sha256Buffer(priorContent),
|
||||
priorContent, hadFile: true
|
||||
};
|
||||
} else {
|
||||
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
|
||||
}
|
||||
// Campaign activation captures the actual before image just
|
||||
// before replacing this file. Do not wait on remote reads here.
|
||||
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
|
||||
}
|
||||
plans.push(plan);
|
||||
}
|
||||
@@ -1068,12 +1055,6 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
hadFile = true;
|
||||
}
|
||||
}
|
||||
if (!betaServerSet.has(name) && await exists(targetPath)) {
|
||||
// Capture an initial preview; activation records the actual before image.
|
||||
priorContent = await readFile(targetPath);
|
||||
priorSha = sha256Buffer(priorContent);
|
||||
hadFile = true;
|
||||
}
|
||||
const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath)
|
||||
|| sha256Buffer(await readFile(targetPath)) !== composedSha;
|
||||
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation });
|
||||
@@ -1114,8 +1095,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
);
|
||||
const insertDepFile = db().prepare(
|
||||
`INSERT INTO deployment_files
|
||||
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
);
|
||||
|
||||
const uniqueServers = Array.from(new Set([
|
||||
@@ -1140,7 +1121,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
p.priorSha,
|
||||
p.version.sha256,
|
||||
p.priorContent,
|
||||
p.version.content
|
||||
p.version.content,
|
||||
betaServerSet.has(p.server) ? 0 : 1
|
||||
);
|
||||
if (plansToActivate.has(p)) {
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server,
|
||||
@@ -1159,7 +1141,8 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
|
||||
cp.priorSha,
|
||||
cp.composedSha,
|
||||
cp.priorContent,
|
||||
cp.composedContent
|
||||
cp.composedContent,
|
||||
betaServerSet.has(cp.server) ? 0 : 1
|
||||
);
|
||||
if (cp.requiresActivation) {
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server,
|
||||
|
||||
@@ -11,7 +11,7 @@ import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges }
|
||||
// pre-multi-script legacy rows.
|
||||
|
||||
export class RollbackError extends Error {
|
||||
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back') {
|
||||
constructor(message: string, public readonly kind: 'history-expired' | 'no-such-deployment' | 'not-promote' | 'already-rolled-back' | 'snapshot-pending') {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,7 @@ interface FileRow {
|
||||
sha256_after: string | null;
|
||||
content_before: Buffer | null;
|
||||
content_after: Buffer | null;
|
||||
snapshot_pending: number;
|
||||
}
|
||||
|
||||
interface ScriptTargetRow {
|
||||
@@ -100,7 +101,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
|
||||
|
||||
const fileRows = db()
|
||||
.prepare(
|
||||
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after
|
||||
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after, f.snapshot_pending
|
||||
FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
|
||||
JOIN servers s ON s.name = f.server
|
||||
WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
|
||||
@@ -154,6 +155,9 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
|
||||
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
|
||||
&& row.rel_path === f.rel_path && row.preserve_before_image);
|
||||
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
|
||||
if (f.snapshot_pending) {
|
||||
throw new RollbackError(`${f.server}: rollback snapshot for ${f.rel_path} is awaiting activation confirmation. Reconcile the provisioning operation before rolling back.`, 'snapshot-pending');
|
||||
}
|
||||
const arr = perServer.get(f.server);
|
||||
if (arr) arr.push(f);
|
||||
else perServer.set(f.server, [f]);
|
||||
|
||||
@@ -130,10 +130,14 @@ export function applyStagedFilesForOperation(operationId: string, server: string
|
||||
if (row.source_kind === 'beta') {
|
||||
captureBetaBeforeImage(row, before);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET
|
||||
verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`)
|
||||
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
|
||||
before === null ? null : sha256Buffer(before), row.deployment_file_id);
|
||||
if (row.deployment_file_id && !row.preserve_before_image) {
|
||||
db().prepare(`UPDATE deployment_files SET
|
||||
verb = ?, content_before = ?, sha256_before = ?, snapshot_pending = 0 WHERE id = ?`)
|
||||
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
|
||||
before === null ? null : sha256Buffer(before), row.deployment_file_id);
|
||||
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
|
||||
.run(row.source_id, `${server}: captured rollback snapshot for ${row.rel_path}`);
|
||||
}
|
||||
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id);
|
||||
})();
|
||||
if (row.content === null) {
|
||||
|
||||
@@ -240,8 +240,12 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
|
||||
captureBetaBeforeImage(row, bytes);
|
||||
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
|
||||
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
|
||||
if (row.deployment_file_id && !row.preserve_before_image) {
|
||||
db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=?,snapshot_pending=0 WHERE id=?')
|
||||
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
|
||||
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
|
||||
.run(row.source_id, `${job.server}: captured rollback snapshot for ${row.rel_path}`);
|
||||
}
|
||||
if (row.content === null) removeScript(job.server, row.rel_path);
|
||||
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
|
||||
sha256Buffer(row.content), row.content.length, row.inventory_source);
|
||||
|
||||
@@ -15,7 +15,7 @@ export async function GET({ params }: { params: { id: string } }) {
|
||||
// it opens the diff viewer.
|
||||
const files = db()
|
||||
.prepare(
|
||||
`SELECT id, server, verb, rel_path, sha256_before, sha256_after,
|
||||
`SELECT id, server, verb, rel_path, sha256_before, sha256_after, snapshot_pending,
|
||||
LENGTH(content_before) AS size_before,
|
||||
LENGTH(content_after) AS size_after,
|
||||
COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state
|
||||
|
||||
@@ -10,7 +10,7 @@ export async function GET({ params }: { params: { id: string; fileId: string } }
|
||||
}
|
||||
const row = db()
|
||||
.prepare(
|
||||
`SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after
|
||||
`SELECT verb, rel_path, sha256_before, sha256_after, content_before, content_after, snapshot_pending
|
||||
FROM deployment_files
|
||||
WHERE deployment_id = ? AND id = ?`
|
||||
)
|
||||
@@ -22,9 +22,11 @@ export async function GET({ params }: { params: { id: string; fileId: string } }
|
||||
sha256_after: string | null;
|
||||
content_before: Buffer | null;
|
||||
content_after: Buffer | null;
|
||||
snapshot_pending: number;
|
||||
}
|
||||
| undefined;
|
||||
if (!row) throw error(404, 'file not found');
|
||||
if (row.snapshot_pending) return json({ error: 'The rollback snapshot is not available yet. It is captured immediately before file activation.' }, { status: 409 });
|
||||
|
||||
return json({
|
||||
verb: row.verb,
|
||||
|
||||
@@ -191,7 +191,7 @@
|
||||
{#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if}
|
||||
{#if preview}
|
||||
<p><strong>{preview.servers.filter(server => server.tested).length} tested servers</strong> and <strong>{preview.servers.filter(server => !server.tested).length} additional eligible servers</strong>.</p>
|
||||
<p>Confirming queues the file changes. Each server waits until empty and restart capacity is available, then installs its files and restarts. Tested servers only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.</p>
|
||||
<p>Confirming creates a campaign and queues the file changes. Each server waits until empty and restart capacity is available, then captures rollback snapshots, installs its files, and restarts. Tested servers keep their pre-beta snapshots and only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.</p>
|
||||
<p class="muted">Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed settings change, you must review again.</p>
|
||||
{#each preview.blockers as blocker}<p class="ui-error" role="alert">{blocker}</p>{/each}
|
||||
<div class="file-list">{#each preview.files as file}<details open><summary>{file.name} · {artifactVersionLabel(file)} · {file.servers.length} servers</summary><p><code>{file.path}</code> · {file.scope} · SHA {file.sha256.slice(0,12)}</p><ul>{#each file.servers as name}<li><ServerIdentity {name}/><span>{preview.servers.find(server => server.name === name)?.tested ? 'Already tested' : 'Additional target · restart when empty'}</span></li>{/each}</ul></details>{/each}</div>
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
sha256_before: string | null; sha256_after: string | null;
|
||||
size_before: number | null; size_after: number | null;
|
||||
apply_state: 'pending' | 'applying' | 'applied' | 'cancelled';
|
||||
snapshot_pending: number;
|
||||
}
|
||||
interface ServerRow {
|
||||
deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number;
|
||||
@@ -97,7 +98,7 @@
|
||||
diffOpen = true; diffLoading = true; diffError = ''; openFile = null;
|
||||
try {
|
||||
const response = await fetch('/api/campaigns/' + id + '/file/' + f.id);
|
||||
if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : 'Could not load this file comparison. Close and select it again to retry.');
|
||||
if (!response.ok) throw new Error(response.status === 410 ? 'File contents expired under the retention policy.' : response.status === 409 ? 'The rollback snapshot is not available yet. Refresh after activation to compare files.' : 'Could not load this file comparison. Close and select it again to retry.');
|
||||
const body = await response.json();
|
||||
if (id === $page.params.id && current === diffRequest) openFile = {id:f.id, ...body};
|
||||
} catch (error) { if (id === $page.params.id && current === diffRequest) diffError = (error as Error).message; }
|
||||
@@ -144,7 +145,7 @@
|
||||
|
||||
<section class="outcome" aria-label="Campaign outcome">
|
||||
<h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2>
|
||||
<p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}</p>
|
||||
<p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. The campaign captures rollback snapshots immediately before applying files, then restarts the server and verifies health. Tested servers keep their original pre-beta snapshots.'}</p>
|
||||
{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>
|
||||
{:else if withdrawn}<p>Withdrawn changes will not be activated by this campaign.</p>
|
||||
{:else}<p>Every recorded server completed activation.</p>{/if}
|
||||
@@ -211,8 +212,9 @@
|
||||
<h3><HistoricalServerIdentity name={srv} deletedAt={detail.servers.find(server => server.server === srv)?.server_deleted_at} /></h3>
|
||||
<div class="file-rows">
|
||||
{#each files as f (f.id)}
|
||||
<button class="file-row" disabled={detail.fileContentsExpired} onclick={() => loadDiff(f)}>
|
||||
<Badge variant={verbVariant(f.verb)} size="sm" uppercase>{f.verb}</Badge>
|
||||
<button class="file-row" disabled={detail.fileContentsExpired || !!f.snapshot_pending} onclick={() => loadDiff(f)}>
|
||||
{#if f.snapshot_pending}<Badge variant="neutral" size="sm">{f.apply_state === 'cancelled' ? 'Snapshot not captured' : 'Snapshot pending'}</Badge>
|
||||
{:else}<Badge variant={verbVariant(f.verb)} size="sm" uppercase>{f.verb}</Badge>{/if}
|
||||
{#if f.apply_state !== 'applied'}<Badge variant="neutral" size="sm">{f.apply_state === 'pending' ? 'Queued' : f.apply_state === 'applying' ? 'Needs inspection' : 'Cancelled'}</Badge>{/if}
|
||||
<span class="path">{f.rel_path}</span>
|
||||
<span class="sizes">
|
||||
@@ -228,7 +230,7 @@
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<OperationHistory steps={campaignFlow(detail.servers, Boolean(detail.deployment.finished_at))} events={detail.events} active={detail.servers.some(s => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'))} />
|
||||
<OperationHistory steps={campaignFlow(detail.servers, Boolean(detail.deployment.finished_at), detail.files)} events={detail.events} active={detail.servers.some(s => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'))} />
|
||||
<Dialog bind:open={diffOpen} title={openFile?.relPath ?? 'File comparison'} wide>
|
||||
{#if diffError}<p class="ui-error" role="alert">{diffError}</p><Button onclick={() => { if (retryFile) void loadDiff(retryFile); }}>Retry comparison</Button>
|
||||
{:else if diffLoading}<EmptyState title="Loading comparison…" />
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { activateFiles } from '../helpers/staged-files.js';
|
||||
|
||||
@@ -13,7 +14,7 @@ beforeEach(async () => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'map' = 'game') {
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
@@ -31,6 +32,92 @@ async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'ma
|
||||
const destination = () => join(fixture.t5, servers[1], 'runtime/plutonium/storage/t5/raw/scripts/sp/review.gsc');
|
||||
|
||||
describe('reviewed beta promotion', () => {
|
||||
it('upgrades existing campaign history without marking known snapshots pending', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy', 'promote', ?)").run(new Date().toISOString());
|
||||
db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before) VALUES ('legacy', ?, 'updated', 'old.gsc', ?)")
|
||||
.run(servers[0], Buffer.from('original'));
|
||||
closeDb();
|
||||
const legacy = new Database(fixture.dbPath);
|
||||
try { legacy.exec('ALTER TABLE deployment_files DROP COLUMN snapshot_pending'); }
|
||||
finally { legacy.close(); }
|
||||
expect(db().prepare('SELECT snapshot_pending, content_before FROM deployment_files').get())
|
||||
.toEqual({ snapshot_pending: 0, content_before: Buffer.from('original') });
|
||||
});
|
||||
it.each(['script', 'baseline'] as const)('queues %s promotion without contacting additional host agents', async kind => {
|
||||
const { betaId } = await ready(kind);
|
||||
const { prepareNativeHost } = await import('../helpers/native-host.js');
|
||||
await prepareNativeHost([servers[1]]);
|
||||
const native = await import('../../src/lib/server/backends/native.js');
|
||||
const request = vi.spyOn(native, 'nativeRequest').mockRejectedValue(new Error('Agent unavailable'));
|
||||
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const result = await promoteBeta(betaId, previewBetaPromotion(betaId).token);
|
||||
expect(result.awaitingActivation).toBe(1);
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT snapshot_pending FROM deployment_files WHERE deployment_id = ? AND server = ?')
|
||||
.get(result.deploymentId, servers[1])).toEqual({ snapshot_pending: 1 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ kind: 'script', before: 'changed while queued' },
|
||||
{ kind: 'baseline', before: 'configuration changed while queued' },
|
||||
{ kind: 'script', before: '' },
|
||||
{ kind: 'script', before: null }
|
||||
] as const)('captures activation-time bytes and restores them: %j', async ({ kind, before }) => {
|
||||
const { betaId } = await ready(kind);
|
||||
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
||||
const path = kind === 'script' ? destination() : join(destOfKind(servers[1], 'config'), configFileOf(servers[1]));
|
||||
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const result = await promoteBeta(betaId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const file = db().prepare('SELECT id FROM deployment_files WHERE deployment_id = ? AND server = ?')
|
||||
.get(result.deploymentId, servers[1]) as { id: number };
|
||||
const { GET: diff } = await import('../../src/routes/api/campaigns/[id]/file/[fileId]/+server.js');
|
||||
const params = { id: result.deploymentId, fileId: String(file.id) };
|
||||
expect((await diff({ params })).status).toBe(409);
|
||||
// An edit after confirmation must be the rollback target, not an older preview.
|
||||
if (before !== null) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, before);
|
||||
}
|
||||
await activateFiles('campaign', result.deploymentId);
|
||||
expect(db().prepare('SELECT snapshot_pending, verb, content_before FROM deployment_files WHERE id = ?').get(file.id))
|
||||
.toEqual({ snapshot_pending: 0, verb: before === null ? 'added' : 'updated', content_before: before === null ? null : Buffer.from(before) });
|
||||
expect(db().prepare('SELECT message FROM campaign_events WHERE deployment_id = ?').all(result.deploymentId))
|
||||
.toContainEqual({ message: `${servers[1]}: captured rollback snapshot for ${kind === 'script' ? 'review.gsc' : configFileOf(servers[1])}` });
|
||||
expect((await (await diff({ params })).json()).before).toBe(before);
|
||||
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
||||
const rollback = await rollbackPromoteDeployment(result.deploymentId);
|
||||
await activateFiles('campaign', rollback.deploymentId);
|
||||
if (before === null) expect(existsSync(path)).toBe(false);
|
||||
else expect(await readFile(path, 'utf8')).toBe(before);
|
||||
});
|
||||
|
||||
it('withdraws unactivated changes without needing snapshots, preserving existing files', async () => {
|
||||
const { betaId } = await ready();
|
||||
await writeFile(destination(), 'untouched');
|
||||
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const result = await promoteBeta(betaId);
|
||||
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
||||
const rollback = await rollbackPromoteDeployment(result.deploymentId);
|
||||
expect(rollback.affectedServers).not.toContain(servers[1]);
|
||||
await activateFiles('campaign', rollback.deploymentId);
|
||||
expect(await readFile(destination(), 'utf8')).toBe('untouched');
|
||||
});
|
||||
|
||||
it('blocks rollback without mutation when activation has not confirmed its snapshot', async () => {
|
||||
const { betaId } = await ready();
|
||||
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const result = await promoteBeta(betaId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE source_id = ?").run(result.deploymentId);
|
||||
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
||||
await expect(rollbackPromoteDeployment(result.deploymentId)).rejects.toMatchObject({ kind: 'snapshot-pending' });
|
||||
expect(db().prepare('SELECT rolled_back_by FROM deployments WHERE id = ?').get(result.deploymentId)).toEqual({ rolled_back_by: null });
|
||||
expect(db().prepare("SELECT COUNT(*) n FROM deployments WHERE action = 'rollback'").get()).toEqual({ n: 0 });
|
||||
});
|
||||
|
||||
it('previews exact versions and additional targets without writing, then links the activation campaign', async () => {
|
||||
const { betaId, script, version } = await ready();
|
||||
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
@@ -42,6 +129,12 @@ describe('reviewed beta promotion', () => {
|
||||
const result = await promoteBeta(betaId, preview.token);
|
||||
expect(result).toMatchObject({ awaitingActivation: 1, failedServers: [] });
|
||||
expect(result.deploymentId).toContain('promote');
|
||||
const { GET: campaign } = await import('../../src/routes/api/campaigns/[id]/+server.js');
|
||||
const summary = await (await campaign({ params: { id: result.deploymentId } })).json();
|
||||
expect(summary.files).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ server: servers[0], snapshot_pending: 0 }),
|
||||
expect.objectContaining({ server: servers[1], snapshot_pending: 1 })
|
||||
]));
|
||||
expect(existsSync(destination())).toBe(false);
|
||||
await activateFiles('campaign', result.deploymentId);
|
||||
expect(await readFile(destination(), 'utf8')).toBe('main(){}');
|
||||
|
||||
@@ -697,7 +697,7 @@ describe('native instance provisioning', () => {
|
||||
const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes');
|
||||
db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString());
|
||||
db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name);
|
||||
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after) VALUES('campaign',?,'updated','hud.gsc',?)").run(job.name, after).lastInsertRowid);
|
||||
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, after).lastInsertRowid);
|
||||
const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
||||
queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId });
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
@@ -714,13 +714,13 @@ describe('native instance provisioning', () => {
|
||||
remote.players = 0; await advance();
|
||||
const activation = (await operationFor(restart.id))!;
|
||||
expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } });
|
||||
expect(db().prepare('SELECT content_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null });
|
||||
expect(db().prepare('SELECT content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 });
|
||||
const files = stage.payload.files as Array<{ id: number; path: string }>;
|
||||
reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } });
|
||||
await advance();
|
||||
expect((await detail(restart.id)).state).toBe('done');
|
||||
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied');
|
||||
expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex') });
|
||||
expect(db().prepare('SELECT content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 });
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
|
||||
});
|
||||
|
||||
@@ -59,8 +59,15 @@ describe('operation progress', () => {
|
||||
expect(campaignFlow([{ restart_state: 'failed', files_changed: 0, restarted_at: null }], false)[0].state).toBe('failed');
|
||||
});
|
||||
it('requires health verification after campaign restart and preserves manual work', () => {
|
||||
expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'current']);
|
||||
expect(campaignFlow([{ restart_state: 'pending', files_changed: 1, restarted_at: 'now' }], false).map(s => s.state)).toEqual(['done', 'done', 'done', 'done', 'current']);
|
||||
expect(campaignFlow([{ restart_state: 'manual', files_changed: 1, restarted_at: null }], true).some(s => s.state === 'current')).toBe(false);
|
||||
expect(campaignFlow([{ restart_state: 'done', files_changed: 1, restarted_at: 'now' }], true).every(s => s.state === 'done')).toBe(true);
|
||||
});
|
||||
it('shows snapshot confirmation before file application and restart', () => {
|
||||
const servers = [{ server: 'test', restart_state: 'pending', files_changed: 1, restarted_at: null }];
|
||||
const files = [{ server: 'test', snapshot_pending: 1, apply_state: 'applying' }];
|
||||
expect(campaignFlow(servers, false, files)[1]).toEqual({ label: 'Capture rollback snapshots', state: 'current' });
|
||||
expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0 }])[2].state).toBe('current');
|
||||
expect(campaignFlow(servers, false, [{ ...files[0], snapshot_pending: 0, apply_state: 'applied' }])[3].state).toBe('current');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user