Defer server file changes until guarded restart activation

Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
This commit is contained in:
2026-09-27 23:06:16 -04:00
parent eb2a852fdc
commit fb48f64de1
36 changed files with 1634 additions and 421 deletions
+11
View File
@@ -390,6 +390,17 @@ RCON parsing (fake UDP socket).
Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility.
Beta deployments, promotions, drift repairs, and rollbacks stage their file changes
in SQLite. Live server files and inventory remain unchanged while waiting for an
empty or stopped server, other operations, and rolling restart capacity. Provisioning
captures rollback snapshots and applies the staged changes immediately before
requesting the restart. Failed file activation blocks that restart. Queued changes
survive application restarts; cancelling an unapplied beta discards its pending
changes. Manual beta rollback holds both file restoration and restart until
**Resume when empty** is selected on its rollback campaign.
Finish pending file campaigns before opening a beta on their servers; this keeps
an older queued change from overwriting the version tested by the new beta.
Deleting a server preserves its name, recorded beta progress and test evidence,
campaign results and file diffs, provisioning and maintenance history, and match
records. Historical entries show **Deleted** and omit live server links and
+4 -2
View File
@@ -180,8 +180,10 @@ servers. Missing evidence at the deadline produces **inconclusive**.
Health monitoring continues after a pass until promotion.
For failures, select hold as failed, pause for investigation, or automatic
rollback. Rollback restores only files touched by the beta. Restarts can wait
for empty servers or remain manual. Promotion is always manual.
rollback. Rollback queues restoration only for files touched by the beta.
Restoration and restart both wait for an empty server and restart capacity.
The manual option holds those changes in a rollback campaign until you select
**Resume when empty**. Promotion is always manual.
Pause holds evaluation and wave advancement. Resume retains completed progress and the
original deadline, but starts new timing intervals. A fresh attempt resets
+7 -7
View File
@@ -8,7 +8,7 @@ interface BetaServer {
export function betaFlow(beta: { state: string; wait_for_health: number; wait_for_match: number; testPlan?: boolean }, servers: BetaServer[]): OperationStep[] {
const stages = ['deploying', 'awaiting-empty', 'restarting'];
const labels = ['Deploy artifacts', 'Wait for empty servers', 'Restart servers'];
const labels = ['Stage file changes', 'Wait for empty servers', 'Apply files and restart'];
if (beta.wait_for_health) { stages.push('awaiting-health'); labels.push('Verify health'); }
if (beta.wait_for_match) { stages.push('awaiting-match'); labels.push('Observe a completed match'); }
if (beta.testPlan) { stages.push('observing'); labels.push('Evaluate test requirements'); }
@@ -30,13 +30,13 @@ export function betaFlow(beta: { state: string; wait_for_health: number; wait_fo
}
export function campaignFlow(servers: { restart_state: string; restarted_at: string | null; files_changed: number }[], finished: boolean): OperationStep[] {
const positions = servers.map(s => s.restart_state === 'done' ? 4
: s.restart_state === 'failed' && !s.files_changed ? 0
: s.restarted_at ? 3 : 1);
const active = servers.filter(s => !['skipped', 'cancelled'].includes(s.restart_state));
const positions = active.map(s => s.restart_state === 'done' ? 4
: s.restarted_at ? 3 : 0);
const current = positions.length ? Math.min(...positions) : finished ? 4 : 0;
return ['Apply file changes', 'Wait for empty servers', 'Restart servers', 'Verify health'].map((label, i) => ({
return ['Wait for empty servers and restart capacity', 'Apply file changes', 'Restart servers', 'Verify health'].map((label, i) => ({
label, state: i < current ? 'done' : i !== current ? 'pending'
: servers.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: servers.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
: active.some((s, j) => s.restart_state === 'failed' && positions[j] === i) ? 'failed'
: active.some(s => ['manual', 'held'].includes(s.restart_state)) ? 'pending' : 'current'
}));
}
+32 -2
View File
@@ -343,6 +343,35 @@ CREATE TABLE IF NOT EXISTS deployment_files (
CREATE INDEX IF NOT EXISTS idx_deployment_files_dep ON deployment_files(deployment_id);
-- File payloads stay in SQLite until Provisioning has admitted their restart.
-- Existing campaigns/betas without rows here already wrote their files in the
-- previous release; they remain restart-only work during upgrade.
CREATE TABLE IF NOT EXISTS staged_file_changes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_kind TEXT NOT NULL CHECK (source_kind IN ('campaign','beta')),
source_id TEXT NOT NULL,
server TEXT NOT NULL,
kind TEXT NOT NULL,
rel_path TEXT NOT NULL,
content BLOB,
script_version_id INTEGER,
inventory_source TEXT NOT NULL,
deployment_file_id INTEGER REFERENCES deployment_files(id) ON DELETE CASCADE,
preserve_before_image INTEGER NOT NULL DEFAULT 0,
check_before INTEGER NOT NULL DEFAULT 0,
expected_before_sha TEXT,
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','applying','applied','cancelled')),
applied_at TEXT,
UNIQUE (source_kind, source_id, server, kind, rel_path)
);
CREATE INDEX IF NOT EXISTS idx_staged_files_source ON staged_file_changes(source_kind, source_id, server);
CREATE TRIGGER IF NOT EXISTS staged_files_campaign_delete AFTER DELETE ON deployments BEGIN
DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = OLD.id;
END;
CREATE TRIGGER IF NOT EXISTS staged_files_beta_delete AFTER DELETE ON betas BEGIN
DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = CAST(OLD.id AS TEXT);
END;
-- Per-script rows on aggregate promote/rollback deployments. For legacy
-- single-script deployments this may also be backfilled from the deprecated
-- deployments.script_id / script_version_id / prior_version_id columns.
@@ -550,10 +579,11 @@ CREATE TABLE IF NOT EXISTS campaign_events (
message TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_campaign_events ON campaign_events(deployment_id, id);
CREATE TRIGGER IF NOT EXISTS campaign_activity_file AFTER INSERT ON deployment_files
DROP TRIGGER IF EXISTS campaign_activity_file;
CREATE TRIGGER campaign_activity_file AFTER INSERT ON deployment_files
BEGIN
INSERT INTO campaign_events (deployment_id, message)
VALUES (NEW.deployment_id, NEW.server || ': ' || NEW.verb || ' ' || NEW.rel_path);
VALUES (NEW.deployment_id, NEW.server || ': recorded file change — ' || NEW.verb || ' ' || NEW.rel_path);
END;
CREATE TRIGGER IF NOT EXISTS campaign_activity_open AFTER INSERT ON deployments
BEGIN
+142 -241
View File
@@ -12,11 +12,11 @@ async function exclusive<T>(id: number, action: () => Promise<T>): Promise<T> {
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join, posix } from 'node:path';
import { join, posix } from 'node:path';
import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { sha256Buffer } from './files.js';
import { ensureDest, getServer, heldByBeta } from './servers.js';
import { getServer, heldByBeta } from './servers.js';
import {
getBaselineForScope,
getScript,
@@ -30,7 +30,8 @@ import {
type ScriptVersionRow
} from './scripts.js';
import { composeServerCfg } from './config-compose.js';
import { recordScript, removeScript, getServerScript } from './inventory.js';
import { getServerScript, removeScript } from './inventory.js';
import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js';
import { emit } from '../pipeline/bus.js';
import type { BetaStage } from '../pipeline/bus.js';
@@ -207,6 +208,19 @@ function nowIso(): string {
return new Date().toISOString();
}
function unresolvedCampaignWrites(server: string): Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }> {
return db().prepare(`SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path
FROM staged_file_changes f
JOIN deployment_servers ds ON ds.deployment_id = f.source_id AND ds.server = f.server
LEFT JOIN server_deployments operation ON operation.id = ds.provisioning_id
WHERE f.source_kind = 'campaign' AND f.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held','manual')
AND (f.state = 'applying' OR (f.state = 'pending'
AND (ds.restart_state IN ('held','manual') OR operation.id IS NULL
OR operation.state IN ('queued','waiting','running','awaiting-health'))))`)
.all(server) as Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }>;
}
// ---------- Reads ----------
export function getBeta(id: number): BetaRow | undefined {
@@ -370,6 +384,17 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
throw new BetaError('one or more servers are out of scope', 'out-of-scope', outOfScope);
}
// A beta outranks campaigns during activation. Do not let it overtake older
// staged writes which would otherwise overwrite the tested version on closure.
// Legacy campaigns have no staged payload and retain their restart-only priority.
const queuedCampaigns = input.servers.flatMap(unresolvedCampaignWrites);
if (queuedCampaigns.length) {
const interrupted = queuedCampaigns.some(row => row.state === 'applying');
throw new BetaError(interrupted
? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.'
: 'Finish or resume queued file campaigns before opening a beta on these servers.', 'conflict', queuedCampaigns);
}
// Per-(server, script) conflict check. heldByBeta covers "another running
// beta already claimed this target"; the previous stale-.OLD check is no
// longer relevant now that snapshots live in the DB.
@@ -383,14 +408,6 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
}
if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts);
// Ensure destinations exist. The `ensureDest` helper covers only the
// script destination today; config/plugin destinations are created
// on-demand via mkdir during the write loop.
for (const name of input.servers) {
const ok = await ensureDest(name, true);
if (!ok.ok) throw new BetaError(`dest not usable for ${name}: ${ok.reason}`, 'dest-fail');
}
// DB writes in one transaction.
const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols
const insertBeta = db().prepare(
@@ -430,13 +447,12 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map);
}
for (const r of resolved) {
const prior = getServerScript(name, r.script.target_rel_path);
insertBetaServerScript.run(
betaId,
name,
r.script.id,
prior ? 1 : 0,
prior?.script_version_id ?? null
0,
null
);
}
}
@@ -484,133 +500,61 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
await writeFile(join(archive, base), r.version.content);
}
const insertSnapshot = db().prepare(
`INSERT INTO beta_file_snapshots (beta_id, server, rel_path, prior_content, prior_sha256, prior_size)
VALUES (?, ?, ?, ?, ?, ?)`
);
// On-disk deploy per server. If any write fails on a server, mark the
// whole server 'failed' — we can't restart a partially deployed server
// safely. Remaining artifacts on that server are skipped.
const updateStage = db().prepare(
`UPDATE beta_servers
SET stage = ?, error = ?, deployed_at = ?, updated_at = ?
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
);
// Persist the requested bytes without touching any live destination. Provisioning
// captures the original file and inventory immediately before guarded activation.
for (const name of names) {
if (!isActiveBetaServer(betaId, name)) continue;
db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?")
.run(nowIso(), betaId, name);
const serverRow = serverRows.find((r) => r.name === name)!;
let firstErr: string | null = null;
// 1. Non-collapsing artifacts: script/plugin — write directly to the
// target filename. Pre-beta bytes go into beta_file_snapshots so
// rollback can restore them.
for (const { script, version } of nonCollapsing) {
if (firstErr) break;
const dest = destOfKind(name, script.kind);
const targetPath = join(dest, script.target_rel_path);
try {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (existsSync(targetPath)) {
const priorBytes = await readFile(targetPath);
if (!isActiveBetaServer(betaId, name)) break;
insertSnapshot.run(
betaId,
name,
script.target_rel_path,
priorBytes,
sha256Buffer(priorBytes),
priorBytes.byteLength
);
db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id);
}
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path);
if (!isActiveBetaServer(betaId, name)) break;
await writeFile(targetPath, version.content);
if (!isActiveBetaServer(betaId, name)) break;
recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`);
} catch (err) {
firstErr = `${script.name}: ${(err as Error).message}`;
}
}
// 2. Collapsing layer: compose the effective dedicated_sp.cfg from
// baseline + all applicable overlays, write in place. Pre-beta cfg
// bytes go into beta_file_snapshots keyed on the target rel_path.
const hasCollapsing = collapsingBaseline || collapsingOverlays.length > 0;
if (!firstErr && hasCollapsing) {
try {
const changes: Parameters<typeof queueFileChange>[0][] = nonCollapsing.map(({ script, version }) => ({
sourceKind: 'beta' as const, sourceId: String(betaId), server: name,
kind: script.kind, relPath: script.target_rel_path, content: version.content,
scriptVersionId: version.id, inventorySource: `beta:${betaId}`
}));
if (collapsingBaseline || collapsingOverlays.length > 0) {
const composedResult = composeCfgForServer(name, serverRow.game, serverRow.map, {
inBetaBaseline: collapsingBaseline?.version,
inBetaOverlays: collapsingOverlays
});
if (!composedResult) {
throw new Error(
`no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays`
);
}
const dest = destOfKind(name, 'config');
await mkdir(dest, { recursive: true });
const targetRel = configFileOf(name);
const targetPath = join(dest, targetRel);
if (existsSync(targetPath)) {
const priorBytes = await readFile(targetPath);
if (!isActiveBetaServer(betaId, name)) continue;
insertSnapshot.run(
betaId,
name,
targetRel,
priorBytes,
sha256Buffer(priorBytes),
priorBytes.byteLength
);
throw new Error(`no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays`);
}
const composedBuf = Buffer.from(composedResult.text, 'utf8');
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel);
if (!isActiveBetaServer(betaId, name)) continue;
await writeFile(targetPath, composedBuf);
if (!isActiveBetaServer(betaId, name)) continue;
recordScript(
name,
targetRel,
null,
sha256Buffer(composedBuf),
composedBuf.byteLength,
`beta:${betaId}`
);
// Archive the composed preview for operator visibility.
changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name,
kind: 'config', relPath: configFileOf(name), content: composedBuf,
scriptVersionId: null, inventorySource: `beta:${betaId}` });
await writeFile(join(archive, `${name}.${configFileOf(name)}`), composedBuf);
} catch (err) {
firstErr = `configs: ${(err as Error).message}`;
}
}
const t = nowIso();
if (!isActiveBetaServer(betaId, name)) continue;
if (firstErr) {
updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` });
} else {
updateStage.run('awaiting-empty', null, t, t, betaId, name);
db().transaction(() => {
for (const change of changes) queueFileChange(change);
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL")
.run(nowIso(), betaId, name);
})();
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null });
} catch (error) {
const message = `deploy: ${(error as Error).message}`;
db().prepare("UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL")
.run(message, nowIso(), betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: message });
}
}
}
// ---------- rollback ----------
export interface RollbackResult {
betaId: number;
deploymentId: string | null;
restored: string[];
dropped: string[];
warnings: string[];
}
export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
return exclusive(betaId, () => rollbackBetaInternal(betaId));
export async function rollbackBeta(betaId: number, options: { restart?: 'manual' | 'when-empty' } = {}): Promise<RollbackResult> {
return exclusive(betaId, () => rollbackBetaInternal(betaId, options));
}
async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise<RollbackResult> {
const beta = getBeta(betaId);
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
if (beta.state !== 'running' && beta.state !== 'ready-promote') {
@@ -637,86 +581,77 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
WHERE beta_id = ? AND server = ? AND rel_path = ?`
);
const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = [];
const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?');
const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => {
const staged = isFileApplied('beta', String(betaId), bs.server, kind, path);
if (staged !== null) return staged;
// Existing betas predating staging wrote their files before deployed_at.
return !!changed.get(betaId, bs.server, path) || !!bs.deployed_at;
};
for (const bs of servers) {
if (!isActiveBetaServer(betaId, bs.server)) {
warnings.push(`${bs.server}: original server was deleted; retained history was skipped`);
continue;
}
if (!bs.deployed_at && bs.stage === 'queued') continue;
for (const { script } of bundle) {
if (!isActiveBetaServer(betaId, bs.server)) break;
if (isCollapsingKind(script.kind)) continue; // handled once per server below
if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue;
const dest = destOfKind(bs.server, script.kind);
const targetPath = join(dest, script.target_rel_path);
if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) continue;
const bss = getBetaServerScript(betaId, bs.server, script.id);
if (bss?.had_original) {
const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as
| { prior_content: Buffer; prior_sha256: string; prior_size: number }
| undefined;
if (snap) {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (!isActiveBetaServer(betaId, bs.server)) break;
await writeFile(targetPath, snap.prior_content);
if (!isActiveBetaServer(betaId, bs.server)) break;
if (bss.prior_version_id) {
recordScript(bs.server, script.target_rel_path, bss.prior_version_id, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
} else {
recordScript(bs.server, script.target_rel_path, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
}
restored.add(bs.server);
} else {
if (bss?.had_original && !snap) {
warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`);
continue;
}
} else {
// No prior file — the beta wrote a fresh file. Drop it.
if (existsSync(targetPath)) await rm(targetPath, { force: true });
if (!isActiveBetaServer(betaId, bs.server)) break;
removeScript(bs.server, script.target_rel_path);
dropped.add(bs.server);
restorations.push({ server: bs.server, kind: script.kind, relPath: script.target_rel_path,
content: snap?.prior_content ?? null, versionId: bss?.prior_version_id ?? null });
(snap ? restored : dropped).add(bs.server);
}
}
// Collapsing config layer: single restore for the shared cfg file.
if (!isActiveBetaServer(betaId, bs.server)) continue;
if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) {
const dest = destOfKind(bs.server, 'config');
const targetRel = configFileOf(bs.server);
const targetPath = join(dest, targetRel);
if (hasCollapsing && applied(bs, 'config', targetRel)) {
const snap = getSnapshot.get(betaId, bs.server, targetRel) as
| { prior_content: Buffer; prior_sha256: string; prior_size: number }
| undefined;
if (snap) {
await mkdir(dest, { recursive: true });
if (!isActiveBetaServer(betaId, bs.server)) continue;
await writeFile(targetPath, snap.prior_content);
if (!isActiveBetaServer(betaId, bs.server)) continue;
recordScript(bs.server, targetRel, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
restored.add(bs.server);
} else if (existsSync(targetPath)) {
// No snapshot ⇒ the beta wrote the cfg into a server that had
// no pre-existing file. Remove the beta-written cfg to leave
// the server clean.
await rm(targetPath, { force: true });
if (!isActiveBetaServer(betaId, bs.server)) continue;
removeScript(bs.server, targetRel);
dropped.add(bs.server);
}
restorations.push({ server: bs.server, kind: 'config', relPath: targetRel,
content: snap?.prior_content ?? null, versionId: null });
(snap ? restored : dropped).add(bs.server);
}
}
// Snapshots are single-use — safe to drop now that the disk is restored.
const deploymentId = restorations.length ? `dep-beta-rollback-${betaId}-${Date.now()}` : null;
db().transaction(() => {
cancelStagedFiles('beta', String(betaId));
if (deploymentId) {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)")
.run(deploymentId, nowIso(), storedPlan(betaId) ? 'beta-test-plan' : 'ui', betaId);
for (const server of new Set(restorations.map(change => change.server))) {
db().prepare('INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)')
.run(deploymentId, server, restorations.filter(change => change.server === server).length,
options.restart === 'manual' ? 'held' : 'pending');
}
for (const change of restorations) {
const file = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server,
change.content === null ? 'removed' : 'updated', change.relPath,
change.content === null ? null : sha256Buffer(change.content), change.content);
queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server,
kind: change.kind, relPath: change.relPath, content: change.content,
scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`,
deploymentFileId: Number(file.lastInsertRowid) });
}
}
// The queued restoration owns a durable copy before snapshots are released.
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId);
})();
if (deploymentId) emit({ type: 'deployment-opened', deploymentId, action: 'rollback', servers: [...new Set(restorations.map(change => change.server))] });
// Wipe archive + close out beta.
const archive = betaArchiveOf(betaId);
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
db()
.prepare(`UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?`)
.run(nowIso(), betaId);
// Per-version test-failed transition. Skip a version if (a) another beta
// still gates it at ready-promote/promoted, or (b) it's the currently-
// promoted version of its script (already-live code isn't "failed" by
@@ -742,6 +677,7 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
return {
betaId,
deploymentId,
restored: [...restored].sort(),
dropped: [...dropped].sort(),
warnings
@@ -915,8 +851,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
}
}
// Write files. Track per-plan success and per-server first-error.
const planWritten = new Map<Plan, boolean>();
// Build durable activation work; live inventory changes only at activation.
const serverErr = new Map<string, string>();
// Collapsing config layer: one composed cfg per server. Union of beta
@@ -928,6 +863,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
composedContent: Buffer;
composedSha: string;
hadFile: boolean;
requiresActivation: boolean;
}
const cfgPlans: CfgPlan[] = [];
if (collapsing.length > 0) {
@@ -951,7 +887,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
let priorContent: Buffer | null = null;
let priorSha: string | null = null;
let hadFile = false;
// Beta server: prior cfg bytes were snapshotted at openBeta.
// Beta server: preserve its original bytes captured at activation.
if (betaServerSet.has(name)) {
const snap = getSnapshotForPromote.get(betaId, name, targetRel) as
| { prior_content: Buffer; prior_sha256: string }
@@ -962,59 +898,32 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
hadFile = true;
}
}
if (!hadFile && existsSync(targetPath)) {
// Scope-expansion server (or beta server with no snapshot):
// current on-disk cfg is the "before".
if (!betaServerSet.has(name) && existsSync(targetPath)) {
// Capture an initial preview; activation records the actual before image.
priorContent = await readFile(targetPath);
priorSha = sha256Buffer(priorContent);
hadFile = true;
}
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile });
const requiresActivation = !betaServerSet.has(name) || !existsSync(targetPath)
|| sha256Buffer(await readFile(targetPath)) !== composedSha;
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation });
}
}
// Recheck after asynchronous snapshot reads, before any mutation. Execution uses
// the captured target lists/content so later membership changes never expand it.
checkReview();
for (const p of plans) {
checkReview();
const dest = destOfKind(p.server, p.script.kind);
try {
await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true });
checkReview();
await writeFile(join(dest, p.script.target_rel_path), p.version.content);
checkReview();
planWritten.set(p, true);
} catch (err) {
if (err instanceof BetaError) throw err;
planWritten.set(p, false);
if (!serverErr.has(p.server)) serverErr.set(p.server, `${p.script.name}: ${(err as Error).message}`);
}
}
// Write composed cfg to each target.
const cfgWritten = new Map<CfgPlan, boolean>();
// A scope expansion from another promotion may have been queued while this
// beta was active. Reassert the tested bytes after that older campaign, even
// when they already match the live file, so it cannot overwrite this release.
const queuedByServer = new Map(reviewed.preview.servers.map(server => [server.name, unresolvedCampaignWrites(server.name)]));
const plansToActivate = new Set(plans.filter(p => !betaServerSet.has(p.server)
|| queuedByServer.get(p.server)?.some(row => row.kind === p.script.kind && row.rel_path === p.script.target_rel_path)));
for (const cp of cfgPlans) {
checkReview();
try {
const dest = destOfKind(cp.server, 'config');
await mkdir(dest, { recursive: true });
checkReview();
const targetRel = configFileOf(cp.server);
const targetPath = join(dest, targetRel);
// Beta server already has composed content at targetPath; overwrite
// idempotently. Scope-expansion server: write in place.
await writeFile(targetPath, cp.composedContent);
checkReview();
cfgWritten.set(cp, true);
} catch (err) {
if (err instanceof BetaError) throw err;
cfgWritten.set(cp, false);
const msg = `configs: ${(err as Error).message}`;
if (!serverErr.has(cp.server)) serverErr.set(cp.server, msg);
console.error(`[promote] ${cp.server}/configs failed:`, err);
if (queuedByServer.get(cp.server)?.some(row => row.kind === 'config' && row.rel_path === configFileOf(cp.server))) cp.requiresActivation = true;
}
}
const activationServers = new Set([...Array.from(plansToActivate, plan => plan.server),
...cfgPlans.filter(plan => plan.requiresActivation).map(plan => plan.server)]);
// Build the aggregate deployment.
const depId = promoteDeploymentId(betaId);
const startedAt = nowIso();
@@ -1049,13 +958,11 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
for (const { script, version } of bundle) {
insertDepScript.run(depId, script.id, version.id, script.current_version_id, script.kind);
}
// deployment_files — one row per successful (script, server) write for
// non-collapsing artifacts, plus one row per (server, dedicated_sp.cfg)
// for the composed config layer.
// Record the reviewed change. Scope expansion gets its actual before
// image when activated; tested servers keep their pre-beta history.
for (const p of plans) {
if (!planWritten.get(p)) continue;
const verb = p.hadFile ? 'updated' : 'added';
insertDepFile.run(
const file = insertDepFile.run(
depId,
p.server,
verb,
@@ -1065,19 +972,16 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
p.priorContent,
p.version.content
);
recordScript(
p.server,
p.script.target_rel_path,
p.version.id,
p.version.sha256,
p.version.size,
`promote:${depId}`
);
if (plansToActivate.has(p)) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server,
kind: p.script.kind, relPath: p.script.target_rel_path, content: p.version.content,
scriptVersionId: p.version.id, inventorySource: `promote:${depId}`,
deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) });
}
}
for (const cp of cfgPlans) {
if (!cfgWritten.get(cp)) continue;
const verb = cp.hadFile ? 'updated' : 'added';
insertDepFile.run(
const file = insertDepFile.run(
depId,
cp.server,
verb,
@@ -1087,14 +991,12 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
cp.priorContent,
cp.composedContent
);
recordScript(
cp.server,
configFileOf(cp.server),
null,
cp.composedSha,
cp.composedContent.byteLength,
`promote:${depId}`
);
if (cp.requiresActivation) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server,
kind: 'config', relPath: configFileOf(cp.server), content: cp.composedContent,
scriptVersionId: null, inventorySource: `promote:${depId}`,
deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(cp.server) });
}
}
// deployment_servers — one row per unique server. Beta servers are
// already restarted; scope-only servers still need the sweep to drain
@@ -1106,9 +1008,9 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
continue;
}
const filesChanged =
plans.filter((p) => p.server === name && planWritten.get(p)).length +
cfgPlans.filter((c) => c.server === name && cfgWritten.get(c)).length;
if (betaServerSet.has(name)) {
plans.filter((p) => p.server === name).length +
cfgPlans.filter((c) => c.server === name).length;
if (!activationServers.has(name)) {
const bsRestartedAt = db()
.prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`)
.get(betaId, name) as { restarted_at: string | null } | undefined;
@@ -1124,16 +1026,15 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
})();
db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId);
// Snapshots are single-use — the promoted content is live on disk, so the
// pre-beta bytes are no longer needed.
// The deployment history owns the pre-beta bytes for future rollback.
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
const archive = betaArchiveOf(betaId);
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
const landed = Array.from(new Set([
...plans.filter((p) => planWritten.get(p)).map((p) => p.server),
...cfgPlans.filter((c) => cfgWritten.get(c)).map((c) => c.server)
...plans.map((p) => p.server),
...cfgPlans.map((c) => c.server)
])).sort();
emit({ type: 'beta-state', betaId, state: 'promoted' });
@@ -1144,7 +1045,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
deploymentId: depId,
landed,
scriptIds: bundle.map((b) => b.script.id),
awaitingActivation: reviewed.preview.servers.filter(server => !server.tested && !serverErr.has(server.name)).length,
awaitingActivation: [...activationServers].filter(name => !serverErr.has(name)).length,
failedServers: [...serverErr.keys()]
};
}
+26 -29
View File
@@ -1,21 +1,18 @@
import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { readdir, readFile, stat } from 'node:fs/promises';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { dirname, join, posix, relative, sep } from 'node:path';
import { join, posix, relative, sep } from 'node:path';
import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { sha256Buffer } from './files.js';
import {
getServerScript,
recordScript,
removeScript
} from './inventory.js';
import { getServerScript } from './inventory.js';
import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { selectedZombieMap } from './map-aliases.js';
import { getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js';
import { queueFileChange } from './staged-files.js';
// Destinations drift scans, in a stable order so the UI shows scripts first.
// Baseline shares the config destination — one physical dir, two logical kinds.
@@ -309,8 +306,8 @@ export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
export interface RunDriftResult {
deploymentId: string | null;
// `removed` = untracked files deleted from disk (drift kind='untracked')
// `restored` = missing or existing artifacts installed and tracked
// `removed` and `restored` describe queued changes. Provisioning applies
// them only after the server passes its activation gates.
perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[] }>;
totalRemoved: number;
totalRestored: number;
@@ -381,9 +378,8 @@ export function createDriftRebuildCampaign(
/**
* Detect drift on the requested servers (or the whole fleet) and either
* report the plan (dryRun) or remove untracked files and install managed
* artifacts as a single deployment. The sweep restarts each affected server once
* it's empty and waits for it to come back healthy.
* report the plan (dryRun) or stage the changes as a single deployment.
* Provisioning applies them and restarts each affected server once it's empty.
*/
export async function runDriftDeployment(input: RunDriftInput = {}): Promise<RunDriftResult> {
const { dryRun = false, triggeredBy = 'ui' } = input;
@@ -471,15 +467,19 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
}
}
if (entry.driftKind === 'untracked') {
// Delete a file inventory doesn't know about.
// Capture the reviewed deletion without touching the live tree.
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
await rm(abs, { force: true });
removeScript(s.server, entry.relPath);
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
})();
s.removed.push(entry.relPath);
insertFile.run(depId, s.server, 'removed', entry.relPath, entry.sha256, null, content, null);
} else {
// Install the managed artifact without deleting the target first.
// Persist exact replacement bytes so waiting cannot change the plan.
const written = await resolveManagedEntry(s.server, entry);
if (!written) {
s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
@@ -493,20 +493,17 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
if (err.code === 'ENOENT') return null;
throw err;
});
await mkdir(dirname(abs), { recursive: true });
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
await writeFile(abs, written);
recordScript(
s.server,
entry.relPath,
entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
sha256Buffer(written),
written.byteLength,
`drift:${depId}`
);
s.restored.push(entry.relPath);
insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), before, written);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
})();
s.restored.push(entry.relPath);
}
} catch (err) {
s.skipped.push(`${entry.relPath}: ${(err as Error).message}`);
+29 -50
View File
@@ -1,11 +1,8 @@
import { mkdir, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { destOfKind, type ArtifactKind } from '../config.js';
import type { ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { recordScript, removeScript } from './inventory.js';
import { getVersion } from './scripts.js';
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
@@ -142,58 +139,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
// Group files by server so each server gets exactly one deployment_servers row.
const perServer = new Map<string, FileRow[]>();
const staged = stagedFileChanges('campaign', promoteId);
// Composed baseline rows are physically staged as config files. Prefer the
// explicit link instead of inferring their destination from the library.
const kindForFile = (file: FileRow) => staged.find(row => row.deployment_file_id === file.id)?.kind ?? kindForRelPath(file.rel_path);
for (const f of fileRows) {
// A promotion can still be waiting for empty servers. Withdrawing it
// must not restore or remove files that it never installed. Rows from
// older releases have no staging record and retain their audit meaning.
const kind = kindForFile(f);
const preservesBetaSnapshot = staged.some(row => row.server === f.server && row.kind === kind
&& row.rel_path === f.rel_path && row.preserve_before_image);
if (!preservesBetaSnapshot && isFileApplied('campaign', promoteId, f.server, kind, f.rel_path) === false) continue;
const arr = perServer.get(f.server);
if (arr) arr.push(f);
else perServer.set(f.server, [f]);
}
const affectedServers: string[] = [];
const perServerFailures = new Map<string, string>();
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
for (const [server, files] of perServer) {
if (!active(server)) continue;
try {
for (const f of files) {
if (!active(server)) break;
const kind = kindForRelPath(f.rel_path);
const dest = destOfKind(server, kind);
const abs = join(dest, f.rel_path);
await mkdir(join(dest, dirname(f.rel_path)), { recursive: true });
if (!active(server)) break;
const owningScript = scriptByTarget.get(f.rel_path);
if (f.verb === 'added') {
if (existsSync(abs)) await rm(abs, { force: true });
if (!active(server)) break;
removeScript(server, f.rel_path);
} else if (f.verb === 'updated' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
const priorVer = owningScript?.prior_version_id
? getVersion(owningScript.prior_version_id)
: null;
recordScript(
server,
f.rel_path,
priorVer?.id ?? null,
f.sha256_before!,
f.content_before.byteLength,
`rollback:${newDepId}`
);
} else if (f.verb === 'removed' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`);
}
}
if (active(server)) affectedServers.push(server);
} catch (err) {
perServerFailures.set(server, (err as Error).message);
}
}
// Write the new rollback deployment + mark the original.
const insertDep = db().prepare(
`INSERT INTO deployments
@@ -216,6 +182,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
db().transaction(() => {
cancelStagedFiles('campaign', promoteId);
// Withdrawing this source leaves a shared restart available to other
// campaigns. Provisioning cancels an operation with no eligible sources.
db().prepare(`UPDATE deployment_servers SET restart_state = 'skipped', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).run(promoteId);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
@@ -231,8 +202,8 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (perServerFailures.has(server)) {
insertDepServer.run(newDepId, server, 0, 'failed', perServerFailures.get(server)!);
if (files.some(file => file.verb !== 'added' && file.content_before === null)) {
insertDepServer.run(newDepId, server, 0, 'failed', 'Original file snapshot is missing; cannot restore safely');
continue;
}
insertDepServer.run(newDepId, server, files.length, 'pending', null);
@@ -241,7 +212,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
f.verb === 'added' ? 'removed'
: f.verb === 'removed' ? 'added'
: 'updated';
insertDepFile.run(
const inserted = insertDepFile.run(
newDepId,
server,
invertedVerb,
@@ -251,7 +222,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
f.content_after,
f.content_before
);
const owningScript = scriptByTarget.get(f.rel_path);
const priorVersion = owningScript?.prior_version_id ? getVersion(owningScript.prior_version_id) : null;
queueFileChange({ sourceKind: 'campaign', sourceId: newDepId, server,
kind: kindForFile(f), relPath: f.rel_path,
content: f.verb === 'added' ? null : f.content_before,
scriptVersionId: f.verb === 'updated' ? priorVersion?.id ?? null : null,
inventorySource: `rollback:${newDepId}`, deploymentFileId: Number(inserted.lastInsertRowid) });
}
affectedServers.push(server);
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
// Restore each affected script's current_version_id.
+155
View File
@@ -0,0 +1,155 @@
import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { dirname, relative, resolve, isAbsolute } from 'node:path';
import { destOfKind, serverDir, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { sha256Buffer } from './files.js';
import { getServerScript, recordScript, removeScript } from './inventory.js';
type SourceKind = 'campaign' | 'beta';
interface FileChange {
sourceKind: SourceKind;
sourceId: string;
server: string;
kind: ArtifactKind;
relPath: string;
content: Buffer | null;
scriptVersionId?: number | null;
inventorySource: string;
deploymentFileId?: number;
preserveBeforeImage?: boolean;
expectedBeforeSha?: string | null;
}
export interface StagedFileChange {
id: number; source_kind: SourceKind; source_id: string; server: string;
kind: ArtifactKind; rel_path: string; content: Buffer | null;
script_version_id: number | null; inventory_source: string;
deployment_file_id: number | null; preserve_before_image: number;
check_before: number; expected_before_sha: string | null;
state: 'pending' | 'applying' | 'applied' | 'cancelled'; applied_at: string | null;
}
function targetPath(server: string, kind: ArtifactKind, relPath: string): string {
const root = resolve(destOfKind(server, kind));
const target = resolve(root, relPath);
const rel = relative(root, target);
if (!rel || rel === '..' || rel.startsWith('../') || isAbsolute(rel)) throw new Error('File target must be inside its artifact directory');
return target;
}
/** Capture immutable desired bytes without changing the server or its inventory. */
export function queueFileChange(change: FileChange): void {
targetPath(change.server, change.kind, change.relPath);
db().prepare(`INSERT INTO staged_file_changes (source_kind, source_id, server, kind, rel_path, content,
script_version_id, inventory_source, deployment_file_id, preserve_before_image, check_before, expected_before_sha)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source_kind, source_id, server, kind, rel_path) DO UPDATE SET
content = excluded.content, script_version_id = excluded.script_version_id,
inventory_source = excluded.inventory_source, deployment_file_id = excluded.deployment_file_id,
preserve_before_image = excluded.preserve_before_image, check_before = excluded.check_before,
expected_before_sha = excluded.expected_before_sha
WHERE staged_file_changes.state = 'pending'`)
.run(change.sourceKind, change.sourceId, change.server, change.kind, change.relPath, change.content,
change.scriptVersionId ?? null, change.inventorySource, change.deploymentFileId ?? null,
change.preserveBeforeImage ? 1 : 0, change.expectedBeforeSha !== undefined ? 1 : 0, change.expectedBeforeSha ?? null);
}
export function stagedFileChanges(sourceKind: SourceKind, sourceId: string, server?: string): StagedFileChange[] {
return db().prepare(`SELECT * FROM staged_file_changes WHERE source_kind = ? AND source_id = ?
${server === undefined ? '' : 'AND server = ?'} ORDER BY id`)
.all(sourceKind, sourceId, ...(server === undefined ? [] : [server])) as StagedFileChange[];
}
export function cancelStagedFiles(sourceKind: SourceKind, sourceId: string, server?: string): void {
db().prepare(`UPDATE staged_file_changes SET state = 'cancelled' WHERE source_kind = ? AND source_id = ?
AND state = 'pending' ${server === undefined ? '' : 'AND server = ?'}`)
.run(sourceKind, sourceId, ...(server === undefined ? [] : [server]));
}
export function hasAppliedFiles(sourceKind: SourceKind, sourceId: string, server?: string): boolean {
return stagedFileChanges(sourceKind, sourceId, server).some(row => row.state === 'applied' || row.state === 'applying');
}
/** null identifies legacy work that wrote files before staging was introduced. */
export function isFileApplied(sourceKind: SourceKind, sourceId: string, server: string, kind: ArtifactKind, relPath: string): boolean | null {
const row = stagedFileChanges(sourceKind, sourceId, server).find(row => row.kind === kind && row.rel_path === relPath);
return row ? row.state === 'applied' || row.state === 'applying' : null;
}
function readExisting(path: string): Buffer | null {
try { return readFileSync(path); }
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; }
}
/** Only called after occupancy checks and reservation of the restart slot.
* Synchronous file replacement keeps cancellation, new requests, and deletion
* from interleaving between the final gates and dispatching the restart.
*/
export function applyStagedFilesForOperation(operationId: string, server: string): void {
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
ON s.kind = f.source_kind AND s.source_id = f.source_id AND s.server = f.server
WHERE s.operation_id = ? AND f.server = ? AND f.state IN ('pending','applying') ORDER BY f.id`)
.all(operationId, server) as StagedFileChange[];
if (!rows.length) return;
if (!statSync(serverDir(server)).isDirectory()) throw new Error('Server directory is unavailable');
// Preflight every target before changing any files. An interrupted write
// needs inspection/rollback, never an automatic replay.
const plans = rows.map(row => {
if (row.state === 'applying') throw new Error('A previous file write was interrupted; inspect or roll back the campaign');
const source = row.source_kind === 'campaign'
? db().prepare(`SELECT 1 FROM deployment_servers WHERE deployment_id = ? AND server = ?
AND server_deleted_at IS NULL AND provisioning_id = ? AND restart_state IN ('pending','failed')`).get(row.source_id, server, operationId)
: db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.beta_id = ? AND bs.server = ?
AND bs.server_deleted_at IS NULL AND bs.provisioning_id = ? AND b.state IN ('running','ready-promote')`).get(row.source_id, server, operationId);
if (!source) throw new Error('File change requester is no longer eligible');
const path = targetPath(server, row.kind, row.rel_path);
const before = readExisting(path);
if (row.check_before && (before === null ? null : sha256Buffer(before)) !== row.expected_before_sha) {
throw new Error(`${row.rel_path}: changed while queued; scan and review again`);
}
return { row, path, before };
});
for (const { row, path } of plans) {
// Earlier coalesced campaigns may have just updated the same path.
const before = readExisting(path);
const prior = getServerScript(server, row.rel_path);
db().transaction(() => {
if (row.source_kind === 'beta') {
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, server, row.rel_path, before, sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, server, row.rel_path, row.kind);
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(row.source_id, server, row.rel_path);
}
if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET
verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`)
.run(row.content === null ? 'removed' : before === null ? 'added' : 'updated', before,
before === null ? null : sha256Buffer(before), row.deployment_file_id);
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE id = ?").run(row.id);
})();
if (row.content === null) {
rmSync(path, { force: true });
removeScript(server, row.rel_path);
} else {
mkdirSync(dirname(path), { recursive: true });
// Preserve existing ownership, permissions, and symlink behavior.
// The durable applying checkpoint prevents replay after a partial write.
writeFileSync(path, row.content);
// A queued version can be deleted independently; the staged bytes
// still apply, but its optional inventory link must remain valid.
const version = row.script_version_id !== null && db().prepare('SELECT 1 FROM script_versions WHERE id = ?').get(row.script_version_id)
? row.script_version_id : null;
recordScript(server, row.rel_path, version, sha256Buffer(row.content), row.content.length, row.inventory_source);
}
db().prepare("UPDATE staged_file_changes SET state = 'applied', applied_at = ? WHERE id = ?")
.run(new Date().toISOString(), row.id);
if (row.source_kind === 'campaign') db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(row.source_id, `${server}: applied ${row.rel_path}`);
}
for (const betaId of new Set(rows.filter(row => row.source_kind === 'beta').map(row => row.source_id))) {
db().prepare('UPDATE beta_servers SET deployed_at = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
.run(new Date().toISOString(), new Date().toISOString(), betaId, server);
}
}
+4 -13
View File
@@ -25,19 +25,10 @@ export function betaState(id: number, ready: boolean): void {
}
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server));
const result = await rollbackBeta(id);
if (plan.rollbackRestart === 'when-empty' && servers.length) {
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
db().transaction(() => {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
.run(deploymentId, new Date().toISOString(), id);
for (const server of servers) {
if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
}
})();
activity(id, 'Restored files; rolling restarts will wait for empty servers');
} else activity(id, 'Restored files; manual restarts are required to load them');
const result = await rollbackBeta(id, { restart: plan.rollbackRestart });
if (!result.deploymentId) activity(id, 'Cancelled queued changes; no deployed files need restoration');
else if (plan.rollbackRestart === 'when-empty') activity(id, 'Queued file restoration and rolling restarts; waiting for empty servers');
else activity(id, 'File restoration is held; resume the rollback campaign to apply it with a safe restart');
if (result.warnings.length) activity(id, result.warnings.join('; '));
}
+7 -1
View File
@@ -51,11 +51,17 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
}
// Only not-yet-submitted campaign changes can share an activation. A later
// file deployment must never be credited to an already running restart.
if (source.kind === 'campaign' && !row.restarted_at) {
// Restorations need their own activation. An older campaign can have
// stale file preconditions after a beta; its failure must not strand
// the rollback that would restore the server.
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
if (source.kind === 'campaign' && !row.restarted_at && !rollback) {
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
AND d.state = 'waiting' AND d.submitted_at IS NULL
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback')
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
}
if (!id) {
+13 -2
View File
@@ -3,6 +3,7 @@ import { db } from '../db/index.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
@@ -64,16 +65,26 @@ export async function restartInRollout(server: string, options: { operationId: s
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
const reason = rolloutWaitReason(server, operationId);
if (reason) return reason;
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
// Every gate, including rollout capacity, has cleared. No async work may
// intervene between applying these files and submitting the restart.
try {
applyStagedFilesForOperation(operationId, server);
} catch (err) {
const message = `File activation failed: ${(err as Error).message}`;
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
finishDeployment(operationId, { ok: false, message });
return { kind: 'failed', message };
}
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
+6 -5
View File
@@ -29,12 +29,13 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!maintenanceAllowed(incident.kind)) return 'held';
}
let held = false;
let ready = false;
for (const source of sources) {
if (source.kind === 'campaign') {
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL')
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) return 'ready';
if (['pending','failed'].includes(row.restart_state)) ready = true;
if (['held','manual'].includes(row.restart_state)) held = true;
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
@@ -42,18 +43,18 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true;
if (row.outcome === 'paused') held = true;
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
return 'ready';
ready = true;
}
} else return 'ready';
} else ready = true;
}
return held ? 'held' : 'cancelled';
return held ? 'held' : ready ? 'ready' : 'cancelled';
}
function blocker(server: string, id: string): string | null {
+1 -1
View File
@@ -195,7 +195,7 @@
<div class="fields">
<label>Testing window (hours from opening)<input id="plan-timeoutHours" aria-invalid={!!fieldError('timeoutHours')} aria-describedby="error-timeoutHours" type="number" min="0.01" max="8760" step="any" bind:value={plan.timeoutHours} required /><span class="error" id="error-timeoutHours">{fieldError('timeoutHours') ?? ''}</span></label>
<label>On test failure<select bind:value={plan.failureAction}><option value="fail">Mark failed and hold</option><option value="pause">Pause for investigation</option><option value="rollback">Roll back automatically</option></select></label>
<label>Rollback restarts<select bind:value={plan.rollbackRestart}><option value="when-empty">Rolling restart when empty</option><option value="manual">Restore files; restart manually</option></select></label>
<label>Rollback activation<select bind:value={plan.rollbackRestart}><option value="when-empty">Restore and restart when empty</option><option value="manual">Hold changes for manual resume</option></select></label>
</div>
<p>Waiting for empty servers uses this window. Pausing and retrying do not extend it. Missing evidence at the deadline is inconclusive. Health checks continue until you manually promote.</p>
</div>
+1 -1
View File
@@ -11,7 +11,7 @@ const states: Record<string, [string, StatusTone]> = {
done: ['Completed', 'online'], passed: ['Test passed', 'online'], tested: ['Test passed', 'online'],
promoted: ['Promoted', 'online'], 'rolled-back': ['Rolled back', 'neutral'], cancelled: ['Cancelled', 'neutral'],
failed: ['Failed', 'danger'], 'test-failed': ['Test failed', 'danger'], interrupted: ['Interrupted', 'danger'],
warning: ['Completed with warnings', 'warning'], held: ['Held by beta', 'neutral'],
warning: ['Completed with warnings', 'warning'], held: ['On hold', 'neutral'], skipped: ['Withdrawn', 'neutral'],
manual: ['Manual action needed', 'warning'], paused: ['Paused', 'warning'], inconclusive: ['Missing evidence', 'warning'],
untested: ['Needs testing', 'neutral'], testing: ['Beta active', 'info'], empty: ['No versions', 'neutral'], attention: ['Needs attention', 'warning']
};
+4 -2
View File
@@ -33,7 +33,9 @@ export async function GET({ params }: { params: { id: string } }) {
const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined;
const promotion = campaign ? { ...campaign,
pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n,
pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled','skipped')").get(campaign.id) as { n: number }).n,
failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
return json({ beta, scripts, servers, events, test: testView(id), promotion });
const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1")
.get(id) as { id: string } | undefined;
return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null });
}
+20 -3
View File
@@ -3,6 +3,8 @@ import { db } from '$lib/server/db/index.js';
import { emit, type BetaStage } from '$lib/server/pipeline/bus.js';
import { storedPlan } from '$lib/server/fleet/beta-tests.js';
import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
import { deployBetaServers, getBeta, isActiveBetaServer } from '$lib/server/fleet/beta.js';
import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
function parseId(raw: string): number | null {
const n = Number.parseInt(raw, 10);
@@ -14,18 +16,33 @@ function parseId(raw: string): number | null {
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
const betaId = parseId(params.id);
if (betaId === null) return json({ ok: false, error: 'bad id' }, { status: 400 });
const beta = getBeta(betaId);
if (!beta || !['running', 'ready-promote'].includes(beta.state)) return json({ ok: false, error: 'Beta is closed or missing' }, { status: 409 });
if (storedPlan(betaId)) return json({ ok: false, error: 'Use the test plan retry action to collect fresh evidence' }, { status: 409 });
const body = (await request.json().catch(() => ({}))) as { server?: string };
if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
const row = db()
.prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
.prepare('SELECT stage, deployed_at, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
.get(betaId, body.server) as
| { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
| { stage: string; deployed_at: string | null; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
| undefined;
if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 });
if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
if (!isActiveBetaServer(betaId, body.server)) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 });
if (stagedFileChanges('beta', String(betaId), body.server).some(change => change.state === 'applying')) {
return json({ ok: false, error: 'A file write was interrupted. Inspect the server and roll back before opening a fresh beta.' }, { status: 409 });
}
// Preparation may have failed before any bytes were queued (for example,
// an overlay without a baseline). Retrying must prepare its payload first.
if (!row.deployed_at && !row.restarted_at) {
try { await deployBetaServers(betaId, [body.server]); }
catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 409 }); }
const prepared = db().prepare('SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?')
.get(betaId, body.server) as { stage: string; error: string | null };
if (prepared.stage === 'failed') return json({ ok: false, error: prepared.error }, { status: 409 });
}
// Best-effort resume point: if we already got past restart, go back to
// health check; otherwise back to waiting for the server to be empty.
+12 -2
View File
@@ -1,7 +1,8 @@
import { json } from '@sveltejs/kit';
import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
import { db } from '$lib/server/db/index.js';
import { getBeta } from '$lib/server/fleet/beta.js';
import { deployBetaServers, getBeta } from '$lib/server/fleet/beta.js';
import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
import { parseBetaPlan } from '$lib/beta-plan.js';
@@ -52,7 +53,16 @@ export async function POST({ params, request }: { params: { id: string }; reques
// Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
// operators open another beta if the overall testing window has expired.
if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
if (db().prepare("SELECT 1 FROM beta_servers WHERE beta_id = ? AND error LIKE 'deploy:%'").get(id)) throw new Error('Deployment was incomplete. Roll back before opening a fresh beta');
if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
}
const prepare = db().prepare("SELECT server FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL AND restarted_at IS NULL")
.all(id) as { server: string }[];
if (prepare.length) {
await deployBetaServers(id, prepare.map(row => row.server));
const failed = db().prepare("SELECT error FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL").get(id) as { error: string | null } | undefined;
if (failed) throw new Error(failed.error ?? 'Could not prepare beta files');
}
db().transaction(() => {
const servers = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ?').all(id) as { server: string }[];
for (const { server } of servers) releaseRestartRequest(server, { kind: 'beta', id: String(id) });
+2 -1
View File
@@ -17,7 +17,8 @@ export async function GET({ params }: { params: { id: string } }) {
.prepare(
`SELECT id, server, verb, rel_path, sha256_before, sha256_after,
LENGTH(content_before) AS size_before,
LENGTH(content_after) AS size_after
LENGTH(content_after) AS size_after,
COALESCE((SELECT state FROM staged_file_changes WHERE deployment_file_id = deployment_files.id), 'applied') AS apply_state
FROM deployment_files
WHERE deployment_id = ?
ORDER BY server, rel_path`
@@ -0,0 +1,25 @@
import { json } from '@sveltejs/kit';
import { db } from '$lib/server/db/index.js';
import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
import { kick } from '$lib/server/pipeline/worker.js';
export function POST({ params }: { params: { id: string } }) {
const changed = db().transaction(() => {
const rows = db().prepare(`SELECT ds.server, d.state FROM deployment_servers ds
LEFT JOIN server_deployments d ON d.id = ds.provisioning_id
WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`)
.all(params.id) as { server: string; state: string | null }[];
for (const row of rows) {
if (row.state && ['failed','interrupted','cancelled'].includes(row.state)) {
releaseRestartRequest(row.server, { kind: 'campaign', id: params.id });
}
}
const result = db().prepare(`UPDATE deployment_servers SET restart_state = 'pending', error = NULL
WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`).run(params.id);
if (result.changes) db().prepare('UPDATE deployments SET finished_at = NULL WHERE id = ?').run(params.id);
return result.changes;
})();
if (!changed) return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 });
void kick().catch(err => console.error('[campaign] activation kick failed:', err));
return json({ ok: true }, { status: 202 });
}
+8 -5
View File
@@ -31,6 +31,7 @@
let servers = $state<BetaServer[]>([]);
let events = $state<ActivityEvent[]>([]);
let promotion = $state<Promotion|null>(null);
let rollbackCampaign = $state<{ id: string }|null>(null);
let loading = $state(true), loadError = $state(''), flash = $state(''), success = $state('');
let acting = $state<string|null>(null), request = 0;
let reviewOpen = $state(false), preview = $state<BetaPromotionPreview|null>(null), reviewError = $state('');
@@ -44,7 +45,7 @@
const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length);
const ranked = $derived([...servers].sort((a,b) => (a.stage === 'failed' ? 0 : a.stage === 'ready' ? 2 : 1) - (b.stage === 'failed' ? 0 : b.stage === 'ready' ? 2 : 1) || a.server.localeCompare(b.server)));
const waiting = $derived(ranked.filter(server => server.stage !== 'ready'));
const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check activation on additional servers.' : 'Reuse these files and this plan when you are ready to test again.'
const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.'
: hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.'
: test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.'
: test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.'
@@ -60,6 +61,7 @@
const body = await res.json();
if (current !== request || requestedId !== id) return;
test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null;
rollbackCampaign = body.rollback ?? null;
now = Date.now(); loadError = '';
} catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; }
finally { if (current === request && requestedId === id) loading = false; }
@@ -86,20 +88,20 @@
throw new Error(body.error ?? 'Promotion could not be started.');
}
reviewOpen = false;
success = `Promotion created. ${body.result.awaitingActivation ?? 0} additional servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`;
success = `Promotion created. ${body.result.awaitingActivation ?? 0} servers await activation${body.result.failedServers?.length ? `; ${body.result.failedServers.length} servers need attention` : ''}.`;
await load();
} catch (error) { reviewError = (error as Error).message; }
finally { acting = null; }
}
async function rollback() {
const reject = beta?.state === 'ready-promote';
const restarts = test?.plan.rollbackRestart === 'manual' ? 'Restart servers manually to load the restored files.' : 'Restarts wait for empty servers.';
const restarts = test?.plan.rollbackRestart === 'manual' ? 'Resume the rollback campaign when ready; files stay unchanged until safety checks pass.' : 'Files are restored just before restarting, after safety checks pass.';
if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.filter(server => !server.server_deleted_at).length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return;
acting = 'rollback'; flash = ''; success = '';
try {
const response = await fetch(`/api/betas/${id}/rollback`,{method:'POST'}), body = await response.json();
if (!response.ok || !body.ok) throw new Error(body.error ?? 'Rollback failed.');
success = `Original files restored. ${restarts}`; await load();
success = `Rollback queued. ${restarts}`; await load();
} catch(error) { flash = (error as Error).message; }
finally { acting = null; }
}
@@ -142,6 +144,7 @@
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
{#if closed}<Button href={`/betas/new?reuse=${id}`}>Reuse files and test plan</Button>{/if}
{#if promotion}<Button href={`/campaigns/${encodeURIComponent(promotion.id)}`}>View campaign</Button>{/if}
{#if rollbackCampaign}<Button href={`/campaigns/${encodeURIComponent(rollbackCampaign.id)}`}>View rollback campaign</Button>{/if}
</div>
{#if promotion}<p class="muted">{promotion.pending} servers awaiting activation · {promotion.failed} need attention. Promoting a version does not mean it is already running everywhere.</p>{/if}
{#if flash}<p role="alert" class="ui-error">{flash}</p>{/if}
@@ -166,7 +169,7 @@
{#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if}
{#if preview}
<p><strong>{preview.servers.filter(server => server.tested).length} tested servers</strong> and <strong>{preview.servers.filter(server => !server.tested).length} additional eligible servers</strong>.</p>
<p>Files are installed when you confirm. Tested servers have already restarted. Additional servers restart when empty, then their health is checked. Eligibility is determined by each file's current scope.</p>
<p>Confirming queues the file changes. Each server waits until empty and restart capacity is available, then installs its files and restarts. Tested servers only need another activation if their configuration changed or earlier queued work could replace the tested files. Eligibility follows each file's current scope.</p>
<p class="muted">Reviewed {$timeFormat.dateTime(preview.createdAt)}. If targets, versions, or composed settings change, you must review again.</p>
{#each preview.blockers as blocker}<p class="ui-error" role="alert">{blocker}</p>{/each}
<div class="file-list">{#each preview.files as file}<details open><summary>{file.name} · {artifactVersionLabel(file)} · {file.servers.length} servers</summary><p><code>{file.path}</code> · {file.scope} · SHA {file.sha256.slice(0,12)}</p><ul>{#each file.servers as name}<li><ServerIdentity {name}/><span>{preview.servers.find(server => server.name === name)?.tested ? 'Already tested' : 'Additional target · restart when empty'}</span></li>{/each}</ul></details>{/each}</div>
+28 -2
View File
@@ -20,6 +20,7 @@
id: number; server: string; verb: string; rel_path: string;
sha256_before: string | null; sha256_after: string | null;
size_before: number | null; size_after: number | null;
apply_state: 'pending' | 'applying' | 'applied' | 'cancelled';
}
interface ServerRow {
deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number;
@@ -40,6 +41,8 @@
let detail = $state<Detail | null>(null);
let loadError = $state('');
let resuming = $state(false);
let resumeError = $state('');
let request = 0;
let diffOpen = $state(false);
let diffError = $state('');
@@ -48,7 +51,8 @@
let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null);
let retryFile = $state<FileRow | null>(null);
const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const withdrawn = $derived(detail?.servers.filter(server => ['skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && !['done', 'skipped', 'cancelled'].includes(server.restart_state)).length ?? 0);
const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => {
const rank: Record<string, number> = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 };
@@ -73,6 +77,16 @@
onMount(() => pollVisible(loadDetail, 5000, false));
async function resume() {
resuming = true; resumeError = '';
try {
const response = await fetch(`/api/campaigns/${encodeURIComponent($page.params.id ?? '')}/resume`, { method: 'POST' });
if (!response.ok) throw new Error((await response.json()).error ?? 'Could not resume campaign.');
await loadDetail();
} catch (error) { resumeError = (error as Error).message; }
finally { resuming = false; }
}
async function loadDiff(f: FileRow) {
retryFile = f;
const id = $page.params.id;
@@ -123,10 +137,21 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome">
<h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{withdrawn ? ` · ${withdrawn} withdrawn` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2>
<p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'File changes wait for an empty server and restart capacity. They are applied immediately before restart, then server health is verified.'}</p>
{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>
{:else if withdrawn}<p>Withdrawn changes will not be activated by this campaign.</p>
{:else}<p>Every recorded server completed activation.</p>{/if}
</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Maintenance · <a href={`/maintenance/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
{#if detail.servers.some(server => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))}
<div><Button onclick={resume} loading={resuming} disabled={resuming}>Resume when empty</Button><p>Apply held changes and restart after the safety checks pass.</p></div>
{/if}
{#if resumeError}<p class="err" role="alert">{resumeError}</p>{/if}
<Card padding="lg">
<div class="meta-grid">
<div class="kv"><span class="k">Started</span><span class="v mono">{fmt(detail.deployment.started_at)}</span></div>
@@ -182,6 +207,7 @@
{#each files as f (f.id)}
<button class="file-row" onclick={() => loadDiff(f)}>
<Badge variant={verbVariant(f.verb)} size="sm" uppercase>{f.verb}</Badge>
{#if f.apply_state !== 'applied'}<Badge variant="neutral" size="sm">{f.apply_state === 'pending' ? 'Queued' : f.apply_state === 'applying' ? 'Needs inspection' : 'Cancelled'}</Badge>{/if}
<span class="path">{f.rel_path}</span>
<span class="sizes">
{#if f.size_before !== null}{f.size_before} B{/if}
+10
View File
@@ -0,0 +1,10 @@
/** File/history unit tests simulate the activation leaf. Provisioning tests cover
* the occupancy, capacity, source, and restart gates before reaching this leaf. */
export async function activateFiles(kind: 'beta' | 'campaign', id: number | string): Promise<void> {
const { stagedFileChanges, applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
for (const server of new Set(stagedFileChanges(kind, String(id)).filter(row => row.state === 'pending').map(row => row.server))) {
const operation = requestRestart(server, { kind, id: String(id) });
applyStagedFilesForOperation(operation.id, server);
}
}
+10 -2
View File
@@ -84,9 +84,11 @@ describe('durable beta test plans', () => {
await tick(); await tick();
expect(existsSync(scriptPath(names[1]))).toBe(false);
for (let i = 0; i < 4; i++) await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
expect(existsSync(scriptPath(names[1]))).toBe(false);
expect(getBeta(id)?.state).toBe('running');
expect(listBetaServers(id)[1].stage).toBe('awaiting-empty');
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
for (let i = 0; i < 6; i++) await tick();
expect(getBeta(id)?.state).toBe('ready-promote');
});
@@ -120,6 +122,9 @@ describe('durable beta test plans', () => {
expect(testView(id)?.servers[0].progress[0].current).toBe(25);
expect(testView(id)?.evidence.filter(e => e.kind === 'round_change')).toHaveLength(1);
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(false);
await postTelemetry(gameEvent('heartbeat', 99, 0, names[1]));
await tick();
expect(existsSync(scriptPath(names[1]))).toBe(true);
});
@@ -170,9 +175,12 @@ describe('durable beta test plans', () => {
const { getBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(getBeta(id)?.state).toBe('rolled-back');
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('main() {}');
expect(existsSync(scriptPath(names[1]))).toBe(false);
expect(db().prepare("SELECT server, restart_state FROM deployment_servers").all()).toEqual([{ server: names[0], restart_state: 'pending' }]);
const { sweep } = await import('../helpers/provisioning.js');
await sweep();
expect(await readFile(scriptPath(names[0]), 'utf8')).toBe('original');
});
it('keeps snapshots and holds after a paused failure, without deploying subsequent waves', async () => {
+4
View File
@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
@@ -22,6 +23,7 @@ async function ready(kind: 'script' | 'baseline' = 'script', scope: 'game' | 'ma
scopeType: scope, scopeValue: scope === 'game' ? 't5' : 'zombie_theater', content: Buffer.from(kind === 'script' ? 'main(){}' : 'set sv_voice "1"\n') });
const version = getLatestVersion(script.id)!;
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [servers[0]], waitForHealth: false, waitForMatch: false });
await activateFiles('beta', betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
return { betaId, script, version };
@@ -40,6 +42,8 @@ describe('reviewed beta promotion', () => {
const result = await promoteBeta(betaId, preview.token);
expect(result).toMatchObject({ awaitingActivation: 1, failedServers: [] });
expect(result.deploymentId).toContain('promote');
expect(existsSync(destination())).toBe(false);
await activateFiles('campaign', result.deploymentId);
expect(await readFile(destination(), 'utf8')).toBe('main(){}');
});
it('rejects changed scope membership before creating a campaign or writing extra files', async () => {
+232
View File
@@ -0,0 +1,232 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(servers);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function open(kind: 'script' | 'plugin' | 'baseline' = 'script') {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { script, version } = createScript({ name: `Staged ${kind}`, kind,
targetRelPath: kind === 'script' ? 'nested/test.gsc' : kind === 'plugin' ? 'test.dll' : undefined,
scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'baseline' ? 'set sv_voice "1"\n' : 'beta bytes') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
return { betaId, script, version };
}
describe('staged beta files', () => {
it.each(['script', 'plugin', 'baseline'] as const)('queues %s bytes without creating live directories or recording deployed inventory', async kind => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const dest = destOfKind(servers[0], kind);
await rm(dest, { recursive: true, force: true });
const { betaId } = await open(kind);
const { db } = await import('../../src/lib/server/db/index.js');
expect(existsSync(dest)).toBe(false);
expect(db().prepare('SELECT stage, deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
.toEqual({ stage: 'awaiting-empty', deployed_at: null });
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
expect(db().prepare('SELECT * FROM beta_file_changes').all()).toHaveLength(0);
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
await activateFiles('beta', betaId);
expect(existsSync(dest)).toBe(true);
expect(db().prepare('SELECT deployed_at FROM beta_servers WHERE beta_id = ?').get(betaId))
.toEqual({ deployed_at: expect.any(String) });
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(1);
});
it('cancels an unactivated beta without changing existing files or queuing a restart', async () => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'original');
const { betaId } = await open();
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(await rollbackBeta(betaId)).toMatchObject({ deploymentId: null, restored: [], dropped: [] });
expect(await readFile(path, 'utf8')).toBe('original');
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ state: 'cancelled' })]);
});
it('captures the actual activation before image and queues an unplanned beta rollback', async () => {
const { destOfKind } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'script'), 'nested/test.gsc');
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'request-time bytes');
const { betaId } = await open();
await writeFile(path, 'activation-time bytes');
await activateFiles('beta', betaId);
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await rollbackBeta(betaId);
expect(result).toMatchObject({ deploymentId: expect.any(String), restored: [servers[0]] });
expect(await readFile(path, 'utf8')).toBe('beta bytes');
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM beta_file_snapshots').all()).toHaveLength(0);
await activateFiles('campaign', result.deploymentId!);
expect(await readFile(path, 'utf8')).toBe('activation-time bytes');
});
it('holds manual rollback files until its campaign is resumed', async () => {
const { betaId } = await open();
await activateFiles('beta', betaId);
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await rollbackBeta(betaId, { restart: 'manual' });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(result.deploymentId))
.toEqual({ restart_state: 'held' });
await expect(activateFiles('campaign', result.deploymentId!)).rejects.toThrow('no longer eligible');
const { destOfKind } = await import('../../src/lib/server/config.js');
expect(await readFile(join(destOfKind(servers[0], 'script'), 'nested/test.gsc'), 'utf8')).toBe('beta bytes');
});
it.each([false, true])('prepares missing files before retrying a beta (test plan: %s)', async planned => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta, listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
const { script, version } = createScript({ name: 'Needs baseline', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false, ...(planned ? { testPlan: defaultPlan() } : {}) });
expect(listBetaServers(betaId)[0].stage).toBe('failed');
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
const retry = () => POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
}) });
expect((await retry()).status).toBe(planned ? 400 : 409);
expect(listBetaServers(betaId)[0].stage).toBe('failed');
const baseline = createScript({ name: 'Required baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id);
expect((await retry()).status).toBe(200);
expect(listBetaServers(betaId)[0]).toMatchObject({ stage: 'awaiting-empty', deployed_at: null });
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(stagedFileChanges('beta', String(betaId))).toEqual([expect.objectContaining({ kind: 'config', state: 'pending' })]);
expect(db().prepare('SELECT * FROM server_scripts').all()).toHaveLength(0);
});
it.each([false, true])('rejects an interrupted file write on retry (test plan: %s)', async planned => {
const { betaId } = await open();
const { db } = await import('../../src/lib/server/db/index.js');
const { defaultPlan } = await import('../../src/lib/beta-plan.js');
if (planned) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(defaultPlan()));
db().prepare("UPDATE beta_servers SET stage = 'failed' WHERE beta_id = ?").run(betaId);
db().prepare("UPDATE staged_file_changes SET state = 'applying' WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId));
const { POST } = planned ? await import('../../src/routes/api/betas/[id]/test/+server.js')
: await import('../../src/routes/api/betas/[id]/retry/+server.js');
const result = await POST({ params: { id: String(betaId) }, request: new Request('http://test/retry', {
method: 'POST', body: JSON.stringify(planned ? { action: 'retry' } : { server: servers[0] })
}) });
expect(result.status).toBe(planned ? 400 : 409);
expect((await result.json()).error).toContain('file write was interrupted');
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'failed' });
});
it.each([
{ operationState: null, fileState: 'pending', restartState: 'pending', blocked: true },
{ operationState: 'waiting', fileState: 'pending', restartState: 'pending', blocked: true },
{ operationState: 'failed', fileState: 'pending', restartState: 'pending', blocked: false },
{ operationState: 'cancelled', fileState: 'pending', restartState: 'pending', blocked: false },
{ operationState: 'interrupted', fileState: 'applying', restartState: 'failed', blocked: true },
{ operationState: 'cancelled', fileState: 'pending', restartState: 'held', blocked: true },
{ operationState: 'failed', fileState: 'pending', restartState: 'manual', blocked: true },
{ operationState: null, fileState: null, restartState: 'pending', blocked: false }
])('protects new beta admission from queued campaign work: %j', async ({ operationState, fileState, restartState, blocked }) => {
const { betaId, script } = await open();
await activateFiles('beta', betaId);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const { promoteBeta, openBeta } = await import('../../src/lib/server/fleet/beta.js');
const promotion = await promoteBeta(betaId);
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ? AND server = ?')
.run(restartState, promotion.deploymentId, servers[1]);
if (operationState) {
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
const operation = requestRestart(servers[1], { kind: 'campaign', id: promotion.deploymentId });
db().prepare('UPDATE server_deployments SET state = ? WHERE id = ?').run(operationState, operation.id);
}
if (fileState) db().prepare('UPDATE staged_file_changes SET state = ? WHERE source_kind = ? AND source_id = ?')
.run(fileState, 'campaign', promotion.deploymentId);
else db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").run(promotion.deploymentId);
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const newer = uploadNewVersion(script.id, Buffer.from('newer beta bytes'));
const request = openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
if (blocked) await expect(request).rejects.toMatchObject({ kind: 'conflict', details: [expect.objectContaining({ deploymentId: promotion.deploymentId })] });
else await expect(request).resolves.toMatchObject({ servers: [servers[1]] });
});
it.each(['script', 'baseline'] as const)('keeps the newest %s promotion when another campaign is queued during its beta', async kind => {
const { betaId: olderBeta, script } = await open(kind);
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const newestBytes = kind === 'script' ? 'newest tested bytes' : 'set sv_voice "0"\n';
const newer = uploadNewVersion(script.id, Buffer.from(newestBytes));
const { openBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { betaId: newerBeta } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: newer.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false });
await activateFiles('beta', olderBeta);
await activateFiles('beta', newerBeta);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id IN (?, ?)").run(olderBeta, newerBeta);
const older = await promoteBeta(olderBeta);
const latest = await promoteBeta(newerBeta);
expect(latest.awaitingActivation).toBe(2);
const { stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
expect(stagedFileChanges('campaign', latest.deploymentId, servers[1]))
.toEqual([expect.objectContaining({ preserve_before_image: 1, state: 'pending' })]);
const { requestRestart } = await import('../../src/lib/server/provisioning/index.js');
const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId });
const second = requestRestart(servers[1], { kind: 'campaign', id: latest.deploymentId });
expect(second.id).toBe(first.id);
await activateFiles('campaign', latest.deploymentId);
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]);
expect(await readFile(join(destOfKind(servers[1], kind), relPath), 'utf8')).toContain(newestBytes.trim());
expect(db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
.get(latest.deploymentId, servers[1])).toEqual({ content_before: null });
});
it.each([false, true])('retains the participant pre-beta image when changed composition activation is %s', async activatePromotion => {
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
const path = join(destOfKind(servers[0], 'config'), configFileOf(servers[0]));
await mkdir(dirname(path), { recursive: true });
await writeFile(path, 'original configuration');
const { betaId } = await open('baseline');
await activateFiles('beta', betaId);
const betaBytes = await readFile(path, 'utf8');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { script, version } = createScript({ name: 'Later overlay', kind: 'config', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "0"\n') });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await promoteBeta(betaId);
expect(result.awaitingActivation).toBe(2);
expect(await readFile(path, 'utf8')).toBe(betaBytes);
if (activatePromotion) {
await activateFiles('campaign', result.deploymentId);
expect(await readFile(path, 'utf8')).toContain('set sv_voice "0"');
}
const history = db().prepare('SELECT content_before FROM deployment_files WHERE deployment_id = ? AND server = ?')
.get(result.deploymentId, servers[0]) as { content_before: Buffer };
expect(history.content_before.toString()).toBe('original configuration');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const rollback = await rollbackPromoteDeployment(result.deploymentId);
expect(rollback.affectedServers).toContain(servers[0]);
expect(await readFile(path, 'utf8')).not.toBe('original configuration');
await activateFiles('campaign', rollback.deploymentId);
expect(await readFile(path, 'utf8')).toBe('original configuration');
});
});
+112
View File
@@ -0,0 +1,112 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
let fixture: Fixture;
const server = 't5_zombie_theater_1';
const campaign = 'held-rollback';
const before = Buffer.from('current live bytes');
const restored = Buffer.from('restored version');
const destination = () => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc');
beforeEach(async () => {
vi.resetModules();
vi.clearAllMocks();
fixture = await newFixture([server]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function heldRollback(state = 'held') {
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
await writeFile(destination(), before);
const now = new Date().toISOString();
db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)")
.run(campaign, now, now);
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error)
VALUES (?, ?, 1, ?, 'Waiting for manual activation')`).run(campaign, server, state);
queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: 'rollback.gsc',
content: restored, inventorySource: 'rollback:beta-1' });
}
async function resume(id = campaign) {
const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
return POST({ params: { id } });
}
describe('campaign resume API', () => {
it.each(['held', 'manual'])('queues %s rollback changes without touching live files', async (state) => {
await heldRollback(state);
const { db } = await import('../../src/lib/server/db/index.js');
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
const response = await resume();
expect(response.status).toBe(202);
expect(await response.json()).toEqual({ ok: true });
expect(db().prepare('SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ restart_state: 'pending', error: null, provisioning_id: null });
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)).toEqual({ finished_at: null });
expect(db().prepare('SELECT state, content FROM staged_file_changes WHERE source_id = ?').get(campaign))
.toEqual({ state: 'pending', content: restored });
expect(await readFile(destination())).toEqual(before);
expect(kick).toHaveBeenCalledTimes(1);
expect((await resume()).status).toBe(409);
expect(kick).toHaveBeenCalledTimes(1);
});
it.each(['cancelled', 'failed', 'interrupted'])('releases an old %s operation so the worker creates fresh restart intent', async (state) => {
await heldRollback();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { db } = await import('../../src/lib/server/db/index.js');
const old = requestRestart(server, { kind: 'campaign', id: campaign });
db().prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
.run(state, new Date().toISOString(), old.id);
expect((await resume()).status).toBe(202);
expect(db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ provisioning_id: null });
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
await sweep();
const next = requestRestart(server, { kind: 'campaign', id: campaign });
expect(next.id).not.toBe(old.id);
expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)).toEqual({ state });
expect(db().prepare('SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid').all(campaign))
.toEqual([{ operation_id: old.id }, { operation_id: next.id }]);
expect(await readFile(destination())).toEqual(before);
});
it('retains an existing waiting operation when its held campaign resumes', async () => {
await heldRollback();
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const old = requestRestart(server, { kind: 'campaign', id: campaign });
expect((await resume()).status).toBe(202);
expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id);
});
it.each(['missing', 'done', 'deleted'])('rejects a campaign with %s changes that cannot resume', async (condition) => {
const { db } = await import('../../src/lib/server/db/index.js');
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held');
if (condition === 'deleted') {
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry(); // Reusing its name must not resume old work.
}
const response = await resume();
expect(response.status).toBe(409);
expect(await response.json()).toEqual({ error: 'This campaign has no held changes to resume.' });
expect(kick).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
if (condition !== 'missing') {
expect(await readFile(destination())).toEqual(before);
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign))
.toEqual({ restart_state: condition === 'done' ? 'done' : 'held' });
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign))
.toMatchObject({ finished_at: expect.any(String) });
}
});
});
+11 -4
View File
@@ -3,6 +3,7 @@ import { readFile, mkdir } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
@@ -34,7 +35,7 @@ async function seedFleet(): Promise<void> {
}
describe('config baseline + overlay flow', () => {
it('composes baseline + overlay and writes dedicated_sp.cfg on beta open, restores pre-beta state on rollback', async () => {
it('composes baseline + overlay for activation and queues restoration of the pre-beta state', async () => {
await seedFleet();
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
@@ -70,9 +71,10 @@ describe('config baseline + overlay flow', () => {
waitForMatch: false
});
// 4. Verify: composed cfg lands at configs/dedicated_sp.cfg (in place,
// NO -beta suffix — the game reads the exact filename).
// The live filename is untouched until activation.
const cfgPath = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'dedicated_sp.cfg');
expect(existsSync(cfgPath)).toBe(false);
await activateFiles('beta', betaId);
expect(existsSync(cfgPath)).toBe(true);
const composed = (await readFile(cfgPath)).toString('utf8');
expect(composed).toContain('set g_inactivity "0"');
@@ -82,7 +84,9 @@ describe('config baseline + overlay flow', () => {
// 5. Rollback restores the pre-beta state. There was no cfg on disk
// beforehand, so rollback should just remove the beta-written file.
await rollbackBeta(betaId);
const rollback = await rollbackBeta(betaId);
expect(await readFile(cfgPath, 'utf8')).toBe(composed);
await activateFiles('campaign', rollback.deploymentId!);
expect(existsSync(cfgPath)).toBe(false);
});
@@ -110,6 +114,7 @@ describe('config baseline + overlay flow', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
const row = db()
.prepare(`SELECT stage, error FROM beta_servers WHERE beta_id = ? AND server = ?`)
@@ -146,6 +151,8 @@ describe('plugin kind writes to plugin destination', () => {
});
const pluginPath = join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/plugins', 'demo.dll');
expect(existsSync(pluginPath)).toBe(false);
await activateFiles('beta', betaId);
expect(existsSync(pluginPath)).toBe(true);
const bytes = await readFile(pluginPath);
expect(bytes.equals(Buffer.from([0x00, 0x01, 0x02, 0x03]))).toBe(true);
+5 -1
View File
@@ -645,7 +645,11 @@ describe('server creation map selection', () => {
expect(await scanServerDrift(serverName)).toEqual([]);
const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg');
await rm(mapPath);
expect((await runDriftDeployment({ servers: [serverName] })).totalRestored).toBe(1);
const repair = await runDriftDeployment({ servers: [serverName] });
expect(repair.totalRestored).toBe(1);
expect(existsSync(mapPath)).toBe(false);
const { activateFiles } = await import('../helpers/staged-files.js');
await activateFiles('campaign', repair.deploymentId!);
expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(await scanServerDrift(serverName)).toEqual([]);
});
+6 -1
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
import { gameEvent } from '../helpers/telemetry.js';
@@ -107,6 +108,7 @@ describe('deleted server operational guards', () => {
it('skips deleted campaign targets during restart and rollback while restoring live targets', async () => {
for (const name of names) await writeFile(path(name), 'original content');
const { betaId } = await open();
await activateFiles('beta', betaId);
const { db } = await import('../../src/lib/server/db/index.js');
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
@@ -115,12 +117,15 @@ describe('deleted server operational guards', () => {
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const campaign = await promoteBeta(betaId);
const provisioningBefore = db().prepare('SELECT id FROM server_deployments').all();
await replace(names[1]);
await sweep();
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual(provisioningBefore);
expect(() => requestRestart(names[1], { kind: 'campaign', id: campaign.deploymentId })).toThrow('Restart source no longer exists');
const result = await rollbackPromoteDeployment(campaign.deploymentId);
expect(result.affectedServers).toEqual([names[0]]);
expect(await readFile(path(names[0]), 'utf8')).toBe('beta content');
await activateFiles('campaign', result.deploymentId);
expect(await readFile(path(names[0]), 'utf8')).toBe('original content');
expect(await readFile(path(names[1]), 'utf8')).toBe('replacement content');
expect(db().prepare('SELECT server FROM deployment_servers WHERE deployment_id = ?').all(result.deploymentId)).toEqual([{ server: names[0] }]);
+94 -29
View File
@@ -6,6 +6,28 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
async function applySource(kind: 'campaign' | 'beta', id: string) {
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const table = kind === 'campaign' ? 'deployment_servers' : 'beta_servers';
const key = kind === 'campaign' ? 'deployment_id' : 'beta_id';
const servers = db().prepare(`SELECT server FROM ${table} WHERE ${key} = ?`).all(id) as { server: string }[];
for (const { server } of servers) {
const operation = requestRestart(server, { kind, id });
applyStagedFilesForOperation(operation.id, server);
}
}
// Scanner tests exercise the queued file application separately from the
// Provisioning occupancy/concurrency integration tests.
async function applyDriftDeployment(input: import('../../src/lib/server/fleet/drift.js').RunDriftInput = {}) {
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const result = await runDriftDeployment(input);
if (result.deploymentId) await applySource('campaign', result.deploymentId);
return result;
}
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(['t5_zombie_theater_1']);
@@ -22,27 +44,67 @@ afterEach(async () => {
});
describe('reviewed drift plans', () => {
it('keeps live deletions, replacements, and inventory unchanged until activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'managed plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
targetRelPath: 'managed.dll', content: Buffer.from('new plugin') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const target = join(destOfKind(server, 'plugin'), 'managed.dll');
const stray = join(destOfKind(server, 'config'), 'stray.cfg');
await writeFile(target, 'current plugin');
await writeFile(stray, 'current settings');
const result = await runDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 1, totalRestored: 1 });
expect(await readFile(target, 'utf8')).toBe('current plugin');
expect(await readFile(stray, 'utf8')).toBe('current settings');
expect(getServerScript(server, 'managed.dll')).toBeUndefined();
await applySource('campaign', result.deploymentId!);
expect(await readFile(target, 'utf8')).toBe('new plugin');
expect(existsSync(stray)).toBe(false);
expect(getServerScript(server, 'managed.dll')?.script_version_id).toBe(version.id);
});
it('preserves a file changed while its reviewed deletion waits for activation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'reviewed');
const plan = await previewDrift(['t5_zombie_theater_1']);
const result = await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await writeFile(path, 'new operator settings');
await expect(applySource('campaign', result.deploymentId!)).rejects.toThrow();
expect(await readFile(path, 'utf8')).toBe('new operator settings');
});
it('rejects changed file bytes before any deletion and returns a fresh review', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const path = join(fixture.t5, 't5_zombie_theater_1', 'configs', 'stray.cfg');
await writeFile(path, 'original');
const plan = await previewDrift(['t5_zombie_theater_1']);
await writeFile(path, 'changed');
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('changed');
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
const refreshed = await previewDrift(plan.servers);
expect(refreshed.fingerprint).not.toBe(plan.fingerprint);
expect((await runDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
expect((await applyDriftDeployment({ servers: refreshed.servers, expectedFingerprint: refreshed.fingerprint })).totalRemoved).toBe(1);
});
it('rejects changed replacement bytes and preserves the reviewed existing file', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const { script, version } = createScript({ name: 'Managed config', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
@@ -51,13 +113,13 @@ describe('reviewed drift plans', () => {
await writeFile(path, 'existing settings');
const plan = await previewDrift(['t5_zombie_theater_1']);
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?').run(Buffer.from('set sv_voice "0"\n'), version.id);
await expect(runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await expect(applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
expect(await readFile(path, 'utf8')).toBe('existing settings');
});
it('keeps selected reconciliation within its reviewed server identities', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { previewDrift, runDriftDeployment, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const { previewDrift, DriftPlanChanged } = await import('../../src/lib/server/fleet/drift.js');
const second = join(fixture.t5, 't5_zombie_theater_2', 'configs');
await mkdir(second, { recursive: true });
await seedServerRegistry();
@@ -65,8 +127,8 @@ describe('reviewed drift plans', () => {
await writeFile(firstPath, 'one');
await writeFile(join(second, 'stray.cfg'), 'two');
const plan = await previewDrift(['t5_zombie_theater_1']);
await expect(runDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await runDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
await expect(applyDriftDeployment({ expectedFingerprint: plan.fingerprint })).rejects.toBeInstanceOf(DriftPlanChanged);
await applyDriftDeployment({ servers: plan.servers, expectedFingerprint: plan.fingerprint });
expect(existsSync(firstPath)).toBe(false);
expect(await readFile(join(second, 'stray.cfg'), 'utf8')).toBe('two');
});
@@ -78,7 +140,7 @@ describe('drift scanning', () => {
] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -87,11 +149,11 @@ describe('drift scanning', () => {
const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll';
const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null });
expect(getServerScript(server, relPath)).toBeDefined();
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false);
expect(getServerScript(server, relPath)).toBeUndefined();
});
@@ -99,7 +161,7 @@ describe('drift scanning', () => {
it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -109,13 +171,13 @@ describe('drift scanning', () => {
const content = Buffer.from('promoted content');
const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const inventory = getServerScript(server, relPath);
const path = join(destOfKind(server, kind), relPath);
await writeFile(path, 'changed outside Electric Cherry');
expect(getServerScript(server, relPath)).toEqual(inventory);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(content);
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -123,18 +185,18 @@ describe('drift scanning', () => {
it('compares composed config with disk even when inventory still has the promoted hash', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'changed outside Electric Cherry');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]);
await runDriftDeployment({ servers: [server] });
await applyDriftDeployment({ servers: [server] });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
});
@@ -142,7 +204,7 @@ describe('drift scanning', () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
await seedServerRegistry();
const server = 't5_zombie_theater_1';
@@ -150,13 +212,14 @@ describe('drift scanning', () => {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n'));
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false });
await applySource('beta', String(betaId));
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
const betaBytes = await readFile(path);
const stray = join(fixture.t5, server, 'configs', 'stray.cfg');
await writeFile(stray, 'also deferred');
expect(await scanServerDrift(server)).toEqual([]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
expect(await readFile(path)).toEqual(betaBytes);
expect(await readFile(stray, 'utf8')).toBe('also deferred');
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
@@ -165,7 +228,7 @@ describe('drift scanning', () => {
it('reports an unavailable registered directory without hiding drift on another game root', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const other = 't6_zombie_tranzit_1';
await mkdir(join(fixture.t6, other, 'configs'), { recursive: true });
@@ -173,7 +236,7 @@ describe('drift scanning', () => {
await seedServerRegistry();
await rm(fixture.t5, { recursive: true });
await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable');
await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
await expect(applyDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
expect(existsSync(fixture.t5)).toBe(false);
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]);
@@ -182,7 +245,7 @@ describe('drift scanning', () => {
it('replaces a seeded config once instead of planning both removal and installation', async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
await seedServerRegistry();
@@ -193,7 +256,7 @@ describe('drift scanning', () => {
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
await writeFile(path, 'seeded config');
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind: 'config', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await applyDriftDeployment({ servers: [server] })).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
expect(await scanServerDrift(server)).toEqual([]);
});
@@ -201,7 +264,7 @@ describe('drift scanning', () => {
it.each(['identical', 'different'])('reconciles an existing %s scoped file in one pass with an update audit', async (mode) => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
@@ -215,17 +278,17 @@ describe('drift scanning', () => {
const target = join(destOfKind(server, 'script'), 'zm_spawn_fix.gsc');
await writeFile(target, before);
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'zm_spawn_fix.gsc', driftKind: 'replace' })]);
expect(await runDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await applyDriftDeployment({ servers: [server], dryRun: true })).toMatchObject({ totalRemoved: 0, totalRestored: 1, deploymentId: null });
expect(await readFile(target)).toEqual(before);
expect(getServerScript(server, 'zm_spawn_fix.gsc')).toBeUndefined();
const result = await runDriftDeployment({ servers: [server] });
const result = await applyDriftDeployment({ servers: [server] });
expect(result).toMatchObject({ totalRemoved: 0, totalRestored: 1 });
expect(await readFile(target)).toEqual(content);
expect(getServerScript(server, 'zm_spawn_fix.gsc')?.script_version_id).toBe(version.id);
expect(db().prepare('SELECT verb, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId))
.toEqual([{ verb: 'updated', content_before: before, content_after: content }]);
expect(await scanServerDrift(server)).toEqual([]);
expect((await runDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
expect((await applyDriftDeployment({ servers: [server] })).deploymentId).toBeNull();
});
it('detects untracked files across script, config, and plugin destinations', async () => {
@@ -335,6 +398,7 @@ async function promoteAndOrphan(
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);
@@ -352,7 +416,7 @@ describe('missing artifact detection', () => {
const { openBeta, promoteBeta } = await import(
'../../src/lib/server/fleet/beta.js'
);
const { scanServerDrift, runDriftDeployment } = await import(
const { scanServerDrift } = await import(
'../../src/lib/server/fleet/drift.js'
);
const { db } = await import('../../src/lib/server/db/index.js');
@@ -383,7 +447,7 @@ describe('missing artifact detection', () => {
expect(missing.length).toBe(1);
expect(missing[0].relPath).toBe('hud.gsc');
const result = await runDriftDeployment({
const result = await applyDriftDeployment({
servers: ['t5_zombie_theater_1'],
dryRun: false,
triggeredBy: 'ui'
@@ -420,6 +484,7 @@ describe('missing artifact detection', () => {
waitForHealth: false,
waitForMatch: false
});
await applySource('beta', String(betaId));
db().prepare(
`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`
).run(new Date().toISOString(), betaId);
+10 -2
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
@@ -37,12 +38,14 @@ describe('map selection through config lifecycle', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
expect(await readFile(cfgPath(servers[0]), 'utf8')).toContain('set sv_maprotation "map zombie_theater"');
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?")
.run(new Date().toISOString(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const promoted = await promoteBeta(betaId);
expect(new Set(promoted.landed)).toEqual(new Set(servers));
await activateFiles('campaign', promoted.deploymentId);
const ascension = await readFile(cfgPath(servers[1]), 'utf8');
expect(ascension).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(ascension).toContain('set sv_voice "1"');
@@ -51,6 +54,7 @@ describe('map selection through config lifecycle', () => {
await rm(cfgPath(servers[1]));
const repaired = await runDriftDeployment({ servers: [servers[1]] });
expect(repaired.totalRestored).toBe(1);
await activateFiles('campaign', repaired.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(ascension);
expect(await scanServerDrift(servers[1])).toEqual([]);
});
@@ -75,11 +79,14 @@ describe('map selection through config lifecycle', () => {
scripts: [{ scriptId: overlay.id, scriptVersionId: getLatestVersion(overlay.id)!.id }],
servers: [servers[1]], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
const preview = await readFile(cfgPath(servers[1]), 'utf8');
expect(preview).toContain('set sv_maprotation "map zombie_cosmodrome"');
expect(preview).toContain('set g_inactivity "0"');
expect(preview).not.toContain('map zombie_moon');
await rollbackBeta(betaId);
const rollback = await rollbackBeta(betaId);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(preview);
await activateFiles('campaign', rollback.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(oldConfig);
});
@@ -90,7 +97,8 @@ describe('map selection through config lifecycle', () => {
await writeFile(join(fixture.t5, servers[1], 'configs', 'stray.cfg'), 'set stray "1"');
const drift = await scanServerDrift(servers[1]);
expect(drift.map((entry) => entry.relPath)).toEqual(['stray.cfg']);
await runDriftDeployment({ servers: [servers[1]] });
const driftResult = await runDriftDeployment({ servers: [servers[1]] });
await activateFiles('campaign', driftResult.deploymentId!);
expect(await readFile(cfgPath(servers[1]), 'utf8')).toBe(defaults);
});
});
+1 -1
View File
@@ -51,7 +51,7 @@ describe('operation progress', () => {
const server = { stage: 'awaiting-empty', deployed_at: 'now', restarted_at: null, healthy_at: null, match_seen_at: null };
it('uses the slowest beta server and omits disabled gates', () => {
const flow = betaFlow({ state: 'running', wait_for_health: 0, wait_for_match: 0 }, [server, { ...server, stage: 'ready' }]);
expect(flow.map(s => s.label)).toEqual(['Deploy artifacts', 'Wait for empty servers', 'Restart servers', 'Review and promote']);
expect(flow.map(s => s.label)).toEqual(['Stage file changes', 'Wait for empty servers', 'Apply files and restart', 'Review and promote']);
expect(flow.map(s => s.state)).toEqual(['done', 'current', 'pending', 'pending']);
});
it('does not mark a failed deployment as completed', () => {
+128
View File
@@ -0,0 +1,128 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const server = 't5_zombie_theater_1';
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([server]);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); });
async function promotion(kind: 'script' | 'plugin' | 'baseline', staged: boolean) {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { destOfKind } = await import('../../src/lib/server/config.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const before = Buffer.from(kind === 'baseline' ? 'sv_voice 0\n' : 'original');
const after = Buffer.from(kind === 'baseline' ? 'sv_voice 1\n' : 'promoted');
const relPath = kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc';
const { script, version: prior } = createScript({ name: 'managed', kind, targetRelPath: relPath, scopeType: 'game', scopeValue: 't5', content: before });
const version = uploadNewVersion(script.id, after);
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('promote', 'promote', ?, 'ui')").run(new Date().toISOString());
db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('promote', ?, 1, 'pending')").run(server);
db().prepare("INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES ('promote', ?, ?, ?, ?)")
.run(script.id, version.id, prior.id, kind);
const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES ('promote', ?, 'updated', ?, ?, ?, ?, ?)`).run(server, relPath, sha256Buffer(before), sha256Buffer(after), before, after);
const path = join(destOfKind(server, kind), relPath);
await mkdir(dirname(path), { recursive: true });
await writeFile(path, staged ? before : after);
if (staged) queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind, relPath, content: after,
scriptVersionId: version.id, inventorySource: 'promote:promote', deploymentFileId: Number(file.lastInsertRowid) });
return { path, before, after, script, prior };
}
async function applyCampaign(id: string) {
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js');
const operation = requestRestart(server, { kind: 'campaign', id });
applyStagedFilesForOperation(operation.id, server);
return operation;
}
describe('deferred promotion rollback', () => {
it('withdraws an unapplied promotion without restoring or restarting its server', async () => {
const { path, before } = await promotion('script', true);
const { db } = await import('../../src/lib/server/db/index.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { applyStagedFilesForOperation, isFileApplied } = await import('../../src/lib/server/fleet/staged-files.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const operation = requestRestart(server, { kind: 'campaign', id: 'promote' });
const rollback = await rollbackPromoteDeployment('promote');
expect(rollback.affectedServers).toEqual([]);
expect(await readFile(path)).toEqual(before);
expect(isFileApplied('campaign', 'promote', server, 'script', 'managed.gsc')).toBe(false);
expect(db().prepare("SELECT restart_state FROM deployment_servers WHERE deployment_id = 'promote'").get()).toEqual({ restart_state: 'skipped' });
expect(db().prepare('SELECT * FROM deployment_servers WHERE deployment_id = ?').all(rollback.deploymentId)).toEqual([]);
applyStagedFilesForOperation(operation.id, server);
expect(await readFile(path)).toEqual(before);
});
it.each(['script', 'plugin', 'baseline'] as const)('queues restoration of applied %s bytes until rollback activation', async (kind) => {
const { path, before, after, prior } = await promotion(kind, true);
await applyCampaign('promote');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const result = await rollbackPromoteDeployment('promote');
expect(result.affectedServers).toEqual([server]);
expect(await readFile(path)).toEqual(after);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
expect(getServerScript(server, kind === 'baseline' ? 'dedicated_sp.cfg' : kind === 'plugin' ? 'managed.dll' : 'managed.gsc')?.script_version_id).toBe(prior.id);
});
it('restores only applied files when a promotion was partly installed', async () => {
const { path, before } = await promotion('script', true);
await applyCampaign('promote');
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const pending = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after)
VALUES ('promote', ?, 'added', 'never.gsc', ?)`).run(server, Buffer.from('queued'));
queueFileChange({ sourceKind: 'campaign', sourceId: 'promote', server, kind: 'script', relPath: 'never.gsc',
content: Buffer.from('queued'), inventorySource: 'promote:promote', deploymentFileId: Number(pending.lastInsertRowid) });
const result = await rollbackPromoteDeployment('promote');
expect(db().prepare('SELECT rel_path FROM deployment_files WHERE deployment_id = ?').all(result.deploymentId))
.toEqual([{ rel_path: 'managed.gsc' }]);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
expect(existsSync(join(dirname(path), 'never.gsc'))).toBe(false);
});
it('keeps another campaign eligible when withdrawing a shared activation source', async () => {
const { path, before } = await promotion('script', true);
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES ('other', 'drift', ?, 'ui')").run(new Date().toISOString());
db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('other', ?, 1, 'pending')").run(server);
queueFileChange({ sourceKind: 'campaign', sourceId: 'other', server, kind: 'script', relPath: 'other.gsc',
content: Buffer.from('other campaign'), inventorySource: 'drift:other' });
const shared = requestRestart(server, { kind: 'campaign', id: 'promote' });
expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).toBe(shared.id);
await rollbackPromoteDeployment('promote');
await applyCampaign('other');
expect(await readFile(path)).toEqual(before);
expect(await readFile(join(dirname(path), 'other.gsc'), 'utf8')).toBe('other campaign');
});
it('stages rollback for legacy promotions with no deferred file records', async () => {
const { path, before, after } = await promotion('plugin', false);
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
const result = await rollbackPromoteDeployment('promote');
expect(result.affectedServers).toEqual([server]);
expect(await readFile(path)).toEqual(after);
await applyCampaign(result.deploymentId);
expect(await readFile(path)).toEqual(before);
});
});
+11 -1
View File
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
@@ -62,9 +63,12 @@ describe('editing promoted script scopes', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
return promoteBeta(betaId);
const result = await promoteBeta(betaId);
await activateFiles('campaign', result.deploymentId);
return result;
}
await promote();
@@ -105,6 +109,7 @@ describe('editing promoted script scopes', () => {
scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: ['t5_zombie_theater_1'], waitForHealth: false, waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
const inventory = db().prepare('SELECT * FROM server_scripts').all();
expect(inventory.length).toBeGreaterThan(0);
@@ -143,6 +148,7 @@ describe('promote creates a deployment', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Force all beta_servers to 'ready' and beta to 'ready-promote'.
db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`)
@@ -216,6 +222,7 @@ describe('rollback beta on ready-promote marks version test-failed', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Force ready-promote so the state machine gate is closed.
db().prepare(`UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?`).run(betaId);
@@ -253,10 +260,12 @@ describe('rollback promote deployment', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
db().prepare(`UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?`)
.run(new Date().toISOString(), betaId);
db().prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ?`).run(betaId);
const promote = await promoteBeta(betaId);
await activateFiles('campaign', promote.deploymentId);
// Sanity: script is promoted.
expect(getScript(script.id)!.current_version_id).toBe(v.id);
@@ -329,6 +338,7 @@ describe('multi-script beta', () => {
waitForHealth: false,
waitForMatch: false
});
await activateFiles('beta', betaId);
// Both scripts are joined to the beta.
const bundle = listBetaScripts(betaId);
+440
View File
@@ -0,0 +1,440 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { chmod, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { existsSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('node:fs', async (importOriginal) => {
const original = await importOriginal<typeof import('node:fs')>();
return { ...original, writeFileSync: vi.fn(original.writeFileSync) };
});
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
}));
let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
const relPath = 'staged.gsc';
const before = Buffer.from('main(){ /* current live script */ }');
const after = Buffer.from('main(){ /* reviewed replacement */ }');
type Source = 'campaign' | 'beta';
const destination = (server = servers[0]) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', relPath);
beforeEach(async () => {
vi.resetModules();
vi.resetAllMocks();
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
vi.mocked(writeFileSync).mockImplementation(actualFs.writeFileSync);
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-09-27T15:00:00Z'));
fixture = await newFixture(servers);
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
vi.stubEnv('KOMODO_API_KEY', 'test');
vi.stubEnv('KOMODO_API_SECRET', 'test');
vi.stubEnv('RCON_PASSWORD', 'test');
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(rconStatusResult).mockResolvedValue({
ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
});
});
afterEach(async () => {
await fixture.cleanup();
vi.unstubAllEnvs();
vi.useRealTimers();
});
async function queueChange(source: Source, targets = [servers[0]]) {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const original = createScript({ name: 'Staged script', kind: 'script', targetRelPath: relPath,
scopeType: 'game', scopeValue: 't5', content: before });
const replacement = uploadNewVersion(original.script.id, after);
for (const server of targets) {
await writeFile(destination(server), before);
recordScript(server, relPath, original.version.id, original.version.sha256, before.length, 'fixture');
}
const inventory = getServerScript(targets[0], relPath);
let sourceId: string;
if (source === 'campaign') {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(replacement.id, original.script.id);
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const result = await runDriftDeployment({ servers: targets });
expect(result.deploymentId).toBeTruthy();
sourceId = result.deploymentId!;
} else {
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const result = await openBeta({ scripts: [{ scriptId: original.script.id, scriptVersionId: replacement.id }],
servers: targets, waitForHealth: false, waitForMatch: false });
sourceId = String(result.betaId);
}
return { sourceId, inventory, replacement };
}
async function tick(source: Source) {
const { sweep, betaTick } = await import('../helpers/provisioning.js');
await (source === 'campaign' ? sweep() : betaTick());
}
async function expectLiveUnchanged(inventory: Awaited<ReturnType<typeof queueChange>>['inventory'], server = servers[0]) {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(await readFile(destination(server))).toEqual(before);
expect(getServerScript(server, relPath)).toEqual(inventory);
}
const blockers = ['occupied', 'unknown occupancy', 'missing Komodo', 'missing RCON', 'previous restart', 'rollout capacity'] as const;
describe.each<Source>(['campaign', 'beta'])('%s staged file activation', (source) => {
it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => {
const { db } = await import('../../src/lib/server/db/index.js');
const { config } = await import('../../src/lib/server/config.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' });
if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' });
if (blocker === 'previous restart' || blocker === 'rollout capacity') {
db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)')
.run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString());
}
const { inventory, replacement } = await queueChange(source);
await expectLiveUnchanged(inventory);
await tick(source);
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
// The replacement and its inventory must already be installed when
// Komodo sees the activation request, after every admission gate passes.
Object.assign(config.komodo, { address: 'http://komodo.test' });
Object.assign(config.rcon, { password: 'test' });
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
db().prepare('DELETE FROM restart_rollout').run();
let bytesAtRestart: Buffer | undefined;
let inventoryAtRestart: typeof inventory;
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
bytesAtRestart = await readFile(destination());
inventoryAtRestart = getServerScript(servers[0], relPath);
return { ok: true };
});
await tick(source);
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(bytesAtRestart).toEqual(after);
expect(inventoryAtRestart).toMatchObject({
script_version_id: replacement.id, sha256: replacement.sha256, size: after.length
});
});
it('does not write when its provisioning request is cancelled during the occupancy probe', async () => {
const { inventory } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'")
.get(servers[0]) as { id: string };
cancelProvisioning(operation.id);
return 0;
});
await tick(source);
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0]))
.toEqual({ state: 'cancelled' });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('never applies old staged work to a replacement server created during its occupancy probe', async () => {
await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const replacementBytes = Buffer.from('new server instance');
let replacementInventory: ReturnType<typeof getServerScript>;
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
await seedServerRegistry();
await writeFile(destination(), replacementBytes);
recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement');
replacementInventory = getServerScript(servers[0], relPath);
return 0;
});
await tick(source);
expect(await readFile(destination())).toEqual(replacementBytes);
expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('preserves existing file permissions when activating staged bytes', async () => {
await queueChange(source);
await chmod(destination(), 0o640);
await tick(source);
expect(await readFile(destination())).toEqual(after);
expect((await stat(destination())).mode & 0o777).toBe(0o640);
});
it('applies the exact queued bytes even if artifact storage changes while waiting', async () => {
const { replacement } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE script_versions SET content = ? WHERE id = ?')
.run(Buffer.from('changed after review'), replacement.id);
await tick(source);
expect(await readFile(destination())).toEqual(after);
});
it('keeps the queued payload across an application restart', async () => {
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconGetPlayerCount).mockResolvedValue(2);
const { inventory } = await queueChange(source);
await tick(source);
await expectLiveUnchanged(inventory);
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
vi.resetModules();
await tick(source);
await expectLiveUnchanged(inventory);
const freshRcon = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0);
await tick(source);
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(await readFile(destination())).toEqual(after);
});
it('does not restart or retain rollout capacity when installing staged files fails', async () => {
const { inventory } = await queueChange(source);
const { db } = await import('../../src/lib/server/db/index.js');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
// A directory appearing at the destination causes a deterministic I/O
// failure even when the test process can bypass ordinary file modes.
await rm(destination());
await mkdir(destination());
await tick(source);
expect(komodoRestart).not.toHaveBeenCalled();
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]);
expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ state: 'failed', error: expect.any(String) });
});
});
describe('staged source eligibility', () => {
it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => {
const { sourceId, inventory } = await queueChange('campaign');
const { db } = await import('../../src/lib/server/db/index.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await sweep();
db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId);
await processProvisioning();
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
it('does not install a beta withdrawn after it requests provisioning', async () => {
const { sourceId, inventory } = await queueChange('beta');
const { db } = await import('../../src/lib/server/db/index.js');
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await betaTick();
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId);
await processProvisioning();
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
});
});
describe('staged drift removals and review guards', () => {
it('defers removing a file and its inventory until the empty-server restart', async () => {
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
await writeFile(destination(), before);
recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture');
const inventory = getServerScript(servers[0], relPath);
vi.mocked(rconGetPlayerCount).mockResolvedValue(1);
await runDriftDeployment({ servers: [servers[0]] });
await expectLiveUnchanged(inventory);
await tick('campaign');
await expectLiveUnchanged(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
let fileExistedAtRestart: boolean | undefined;
let inventoryAtRestart: typeof inventory;
vi.mocked(komodoRestart).mockImplementationOnce(async () => {
fileExistedAtRestart = existsSync(destination());
inventoryAtRestart = getServerScript(servers[0], relPath);
return { ok: true };
});
await tick('campaign');
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(fileExistedAtRestart).toBe(false);
expect(inventoryAtRestart).toBeUndefined();
});
it('fails without overwriting a reviewed target changed while its restart was queued', async () => {
const { inventory } = await queueChange('campaign');
const externalChange = Buffer.from('changed outside Electric Cherry while queued');
await writeFile(destination(), externalChange);
await tick('campaign');
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(await readFile(destination())).toEqual(externalChange);
expect(getServerScript(servers[0], relPath)).toEqual(inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0]))
.toMatchObject({ error: expect.stringContaining('changed while queued') });
});
});
describe('shared activation and partial writes', () => {
it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => {
const first = await queueChange('campaign');
const { db } = await import('../../src/lib/server/db/index.js');
const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const second = await runDriftDeployment({ servers: [servers[0]] });
await sweep();
const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[];
expect(linked).toHaveLength(2);
expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1);
db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId);
await processProvisioning();
await expectLiveUnchanged(first.inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' });
db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId);
await tick('campaign');
expect(await readFile(destination())).toEqual(after);
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
.toEqual([{ state: 'applied' }, { state: 'applied' }]);
});
it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => {
const campaign = await queueChange('campaign');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
await sweep();
const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes'));
await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }],
servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
await expectLiveUnchanged(campaign.inventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM betas').all()).toEqual([]);
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId))
.toEqual({ restart_state: 'pending' });
expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId))
.toEqual({ state: 'pending' });
});
it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => {
const beta = await queueChange('beta');
const { db } = await import('../../src/lib/server/db/index.js');
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
// Recreate overlapping work retained from before the admission guard.
// The beta was already open; this older drift still expects its original
// live bytes and must fail independently of the later restoration.
db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)")
.run(new Date(Date.now() - 60_000).toISOString());
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]);
queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script',
relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift',
expectedBeforeSha: sha256Buffer(before) });
await sweep();
await tick('beta');
expect(await readFile(destination())).toEqual(after);
const rollback = await rollbackBeta(Number(beta.sourceId));
vi.setSystemTime(Date.now() + 15_000);
await tick('campaign');
expect(await readFile(destination())).toEqual(before);
expect(komodoRestart).toHaveBeenCalledTimes(2);
const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?')
.get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string };
const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?')
.get(rollback.deploymentId) as { provisioning_id: string };
expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('changed while queued') });
expect(restoration.provisioning_id).not.toBe(old.provisioning_id);
});
it('records partial installation without restarting and never automatically retries the failed write', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after });
const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after });
const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc');
for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) {
await writeFile(path, before);
recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture');
}
const secondInventory = getServerScript(servers[0], 'second.gsc');
await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })),
servers: [servers[0]], waitForHealth: false, waitForMatch: false });
const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
vi.mocked(writeFileSync).mockImplementation((path, content, options) => {
if (path === secondPath) throw new Error('Simulated disk write failure');
actualFs.writeFileSync(path, content, options);
});
await tick('beta');
expect(await readFile(destination())).toEqual(after);
expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id);
expect(await readFile(secondPath)).toEqual(before);
expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory);
expect(komodoRestart).not.toHaveBeenCalled();
expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all())
.toEqual([{ state: 'applied' }, { state: 'applying' }]);
expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get())
.toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') });
expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]);
const attempts = vi.mocked(writeFileSync).mock.calls.length;
await tick('beta');
expect(writeFileSync).toHaveBeenCalledTimes(attempts);
expect(komodoRestart).not.toHaveBeenCalled();
});
});
+15 -3
View File
@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
let fixture: Fixture;
const town = 't6_zombie_town_1';
@@ -39,20 +40,29 @@ describe('T6 config lifecycle in a mixed fleet', () => {
expect(script.target_rel_path).toBe('dedicated_zm.cfg');
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [town], waitForHealth: false, waitForMatch: false });
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
await activateFiles('beta', betaId);
expect(await readFile(cfgPath(town), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(new Date().toISOString(), betaId);
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
const promoted = await promoteBeta(betaId);
expect(new Set(promoted.landed)).toEqual(new Set([town, farm]));
expect(existsSync(cfgPath(farm))).toBe(false);
await activateFiles('campaign', promoted.deploymentId);
const farmConfig = await readFile(cfgPath(farm), 'utf8');
expect(farmConfig).toContain('execgts zm_standard_farm.cfg map zm_transit');
expect(existsSync(join(fixture.t5, t5, 'configs/dedicated_sp.cfg'))).toBe(false);
expect(existsSync(join(fixture.t6, farm, 'configs/dedicated_sp.cfg'))).toBe(false);
expect(await scanServerDrift(farm)).toEqual([]);
await rm(cfgPath(farm));
expect((await runDriftDeployment({ servers: [farm] })).totalRestored).toBe(1);
const drift = await runDriftDeployment({ servers: [farm] });
expect(drift.totalRestored).toBe(1);
expect(existsSync(cfgPath(farm))).toBe(false);
await activateFiles('campaign', drift.deploymentId!);
expect(await readFile(cfgPath(farm), 'utf8')).toBe(farmConfig);
await rollbackPromoteDeployment(promoted.deploymentId);
const rollback = await rollbackPromoteDeployment(promoted.deploymentId);
expect(await readFile(cfgPath(farm), 'utf8')).toBe(farmConfig);
await activateFiles('campaign', rollback.deploymentId);
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
expect(existsSync(cfgPath(farm))).toBe(false);
});
@@ -71,7 +81,9 @@ describe('T6 config lifecycle in a mixed fleet', () => {
content: Buffer.from('sv_voice 0\n') });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [town], waitForHealth: false, waitForMatch: false });
await rollbackBeta(betaId);
await activateFiles('beta', betaId);
const rollback = await rollbackBeta(betaId);
await activateFiles('campaign', rollback.deploymentId!);
expect(await readFile(cfgPath(town), 'utf8')).toBe(previous);
expect(await scanServerDrift(town)).toEqual([]);
await runDriftDeployment({ servers: [town] });