Files
electric-cherry/src/lib/server/fleet/beta.ts
T
josh fb48f64de1 Defer server file changes until guarded restart activation
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely.

Fixes #28

Validation: 805 tests passed, Svelte checks clean, production build passed.
2026-09-27 23:06:16 -04:00

1096 lines
51 KiB
TypeScript

import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { storedPlan } from './beta-tests.js';
import type { BetaPromotionPreview } from '../../beta-promotion.js';
const operations = new Set<number>();
export const betaOperationInFlight = (id: number) => operations.has(id);
async function exclusive<T>(id: number, action: () => Promise<T>): Promise<T> {
if (operations.has(id)) throw new BetaError('A beta operation is already running', 'invalid-state');
operations.add(id);
try { return await action(); } finally { operations.delete(id); }
}
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join, posix } from 'node:path';
import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { sha256Buffer } from './files.js';
import { getServer, heldByBeta } from './servers.js';
import {
getBaselineForScope,
getScript,
getVersion,
listPromotedOverlaysForScope,
listVersions,
serversMatchingScope,
setScope,
type ScopeType,
type ScriptRow,
type ScriptVersionRow
} from './scripts.js';
import { composeServerCfg } from './config-compose.js';
import { getServerScript, removeScript } from './inventory.js';
import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js';
import { emit } from '../pipeline/bus.js';
import type { BetaStage } from '../pipeline/bus.js';
// Whether an artifact writes a dedicated on-disk file (script/plugin) or
// contributes to a composed shared file (config/baseline).
function isCollapsingKind(kind: ArtifactKind): boolean {
return kind === 'config' || kind === 'baseline';
}
// The path we should write to for the composed cfg on this server.
function composedCfgPath(server: string): string {
return join(destOfKind(server, 'config'), configFileOf(server));
}
// Compose the effective dedicated_sp.cfg for a given beta+server, taking
// into account:
// • baseline: from the beta if present, else the currently-promoted
// baseline for the server's scope
// • promoted overlays for the server's scope (excluding scripts being
// replaced by this beta)
// • this beta's overlays whose scope matches the server
//
// Returns null if no baseline is available (nothing to compose against).
interface ComposeInputs {
inBetaBaseline?: ScriptVersionRow;
inBetaOverlays: Array<{ script: ScriptRow; version: ScriptVersionRow }>;
}
function scopeAppliesToServer(
scope: { scope_type: string | null; scope_value: string | null },
server: { game: string | null; map: string | null }
): boolean {
if (!scope.scope_type || !scope.scope_value) return false;
const field = scope.scope_type === 'game' ? server.game : server.map;
return field === scope.scope_value;
}
function composeCfgForServer(
serverName: string,
serverGame: string | null,
serverMap: string | null,
inputs: ComposeInputs
): { text: string; baselineScriptId: number | null } | null {
// Baseline: prefer the beta's baseline if its scope matches this server.
let baselineText: string | null = null;
let baselineScriptId: number | null = null;
if (inputs.inBetaBaseline) {
const s = getScript(inputs.inBetaBaseline.script_id);
if (s && scopeAppliesToServer(s, { game: serverGame, map: serverMap })) {
baselineText = inputs.inBetaBaseline.content.toString('utf8');
baselineScriptId = s.id;
}
}
// Fall back to the currently-promoted baseline for the server's game.
if (baselineText === null && serverGame) {
const promoted = getBaselineForScope('game', serverGame);
if (promoted?.current_version_id) {
const ver = getVersion(promoted.current_version_id);
if (ver) {
baselineText = ver.content.toString('utf8');
baselineScriptId = promoted.id;
}
}
}
if (baselineText === null) return null;
// In-beta script ids that are replacing promoted overlays — those get
// skipped from the promoted overlay list to avoid double-applying an
// older version of the same script.
const inBetaScriptIds = new Set(inputs.inBetaOverlays.map((o) => o.script.id));
// Collect promoted overlays scoped to this server's game AND map.
const promotedOverlays: Array<{ id: number; text: string }> = [];
const promotedIdsSeen = new Set<number>();
const pushIfScoped = (rows: ScriptRow[]) => {
for (const s of rows) {
if (promotedIdsSeen.has(s.id)) continue;
if (inBetaScriptIds.has(s.id)) continue;
if (!scopeAppliesToServer(s, { game: serverGame, map: serverMap })) continue;
if (!s.current_version_id) continue;
const ver = getVersion(s.current_version_id);
if (!ver) continue;
promotedOverlays.push({ id: s.id, text: ver.content.toString('utf8') });
promotedIdsSeen.add(s.id);
}
};
if (serverGame) pushIfScoped(listPromotedOverlaysForScope('game', serverGame));
if (serverMap) pushIfScoped(listPromotedOverlaysForScope('map', serverMap));
// In-beta overlays scoped to this server.
const inBetaScopedOverlays = inputs.inBetaOverlays
.filter(({ script }) => scopeAppliesToServer(script, { game: serverGame, map: serverMap }))
.map(({ script, version }) => ({ id: script.id, text: version.content.toString('utf8') }));
const composed = composeServerCfg(baselineText, [...promotedOverlays, ...inBetaScopedOverlays], serverGame, serverMap, getServer(serverName)?.map_slug);
return { text: composed, baselineScriptId };
}
export interface BetaRow {
id: number;
// Deprecated: the columns are still populated (NOT NULL on legacy schemas)
// from the first bundled script's identity so existing data continues to
// roundtrip, but new code reads beta_scripts for the authoritative list.
script_id: number;
script_version_id: number;
trigger_kind: string;
wait_for_health: number;
wait_for_match: number;
state: 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
opened_at: string;
closed_at: string | null;
notes: string | null;
}
export interface BetaServerRow {
beta_id: number;
server: string;
server_deleted_at: string | null;
had_original: number; // deprecated; per-script data lives in beta_server_scripts
prior_version_id: number | null; // deprecated
stage: BetaStage;
error: string | null;
deployed_at: string | null;
restarted_at: string | null;
provisioning_id: string | null;
healthy_at: string | null;
match_seen_at: string | null;
updated_at: string;
}
export interface BetaScriptRow {
beta_id: number;
script_id: number;
script_version_id: number;
kind: ArtifactKind;
}
export interface BetaServerScriptRow {
beta_id: number;
server: string;
script_id: number;
had_original: number;
prior_version_id: number | null;
}
export class BetaError extends Error {
constructor(
message: string,
public readonly kind:
| 'no-such-script'
| 'no-such-version'
| 'version-script-mismatch'
| 'no-scripts'
| 'no-servers'
| 'unknown-server'
| 'out-of-scope'
| 'conflict'
| 'blocked-backup'
| 'no-such-beta'
| 'not-ready-promote'
| 'not-running'
| 'dest-fail'
| 'invalid-state'
| 'duplicate-script'
| 'preview-changed'
| 'duplicate-target',
public readonly details?: unknown
) {
super(message);
}
}
function nowIso(): string {
return new Date().toISOString();
}
function unresolvedCampaignWrites(server: string): Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }> {
return db().prepare(`SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path
FROM staged_file_changes f
JOIN deployment_servers ds ON ds.deployment_id = f.source_id AND ds.server = f.server
LEFT JOIN server_deployments operation ON operation.id = ds.provisioning_id
WHERE f.source_kind = 'campaign' AND f.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held','manual')
AND (f.state = 'applying' OR (f.state = 'pending'
AND (ds.restart_state IN ('held','manual') OR operation.id IS NULL
OR operation.state IN ('queued','waiting','running','awaiting-health'))))`)
.all(server) as Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }>;
}
// ---------- Reads ----------
export function getBeta(id: number): BetaRow | undefined {
return db().prepare('SELECT * FROM betas WHERE id = ?').get(id) as BetaRow | undefined;
}
export function listBetas(state?: BetaRow['state']): BetaRow[] {
if (state) {
return db().prepare('SELECT * FROM betas WHERE state = ? ORDER BY opened_at DESC').all(state) as BetaRow[];
}
return db().prepare('SELECT * FROM betas ORDER BY opened_at DESC').all() as BetaRow[];
}
export function listBetaServers(betaId: number): BetaServerRow[] {
return db()
.prepare('SELECT * FROM beta_servers WHERE beta_id = ? ORDER BY server')
.all(betaId) as BetaServerRow[];
}
/** History retains names, so operational callers must also check the participant's identity. */
export function isActiveBetaServer(betaId: number, server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server
WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`)
.get(betaId, server);
}
export function listBetaScripts(betaId: number): BetaScriptRow[] {
return db()
.prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id')
.all(betaId) as BetaScriptRow[];
}
export function getBetaServerScript(betaId: number, server: string, scriptId: number): BetaServerScriptRow | undefined {
return db()
.prepare('SELECT * FROM beta_server_scripts WHERE beta_id = ? AND server = ? AND script_id = ?')
.get(betaId, server, scriptId) as BetaServerScriptRow | undefined;
}
// A script is "in a running beta" if beta_scripts joins to a running beta row.
// Existing callers only care about the running-beta id, so return that shape.
export function findRunningBetaForScript(scriptId: number): BetaRow | undefined {
return db()
.prepare(
`SELECT b.* FROM betas b
JOIN beta_scripts bs ON bs.beta_id = b.id
WHERE bs.script_id = ? AND b.state IN ('running','ready-promote')
LIMIT 1`
)
.get(scriptId) as BetaRow | undefined;
}
// ---------- openBeta ----------
export interface OpenBetaScript {
scriptId: number;
scriptVersionId: number;
}
export interface OpenBetaInput {
scripts: OpenBetaScript[];
servers: string[];
testPlan?: BetaPlan;
waitForHealth: boolean;
waitForMatch: boolean;
notes?: string;
}
export interface OpenBetaResult {
betaId: number;
servers: string[];
scripts: OpenBetaScript[];
}
export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
const plan = input.testPlan ? parseBetaPlan(input.testPlan) : null;
if (new Set(input.servers).size !== input.servers.length) throw new BetaError("Select each server only once", "invalid-state");
if (plan?.scope === "quorum" && plan.quorum > input.servers.length) throw new BetaError("Quorum exceeds selected servers", "invalid-state");
if (!input.scripts || input.scripts.length === 0) {
throw new BetaError('at least one script is required', 'no-scripts');
}
if (!input.servers || input.servers.length === 0) {
throw new BetaError('at least one server is required', 'no-servers');
}
// Resolve every (script, version) pair and validate belonging.
interface Resolved { script: ScriptRow; version: ScriptVersionRow }
const resolved: Resolved[] = [];
const seenScriptIds = new Set<number>();
// For non-collapsing kinds (script/plugin), two artifacts targeting the
// same rel_path within one beta is an internal race — reject up front.
// Collapsing kinds (config/baseline) intentionally share a target file
// (dedicated_sp.cfg) and compose, so they skip this check.
const seenTargets = new Map<string, number>();
let baselineInBeta: Resolved | null = null;
for (const s of input.scripts) {
if (seenScriptIds.has(s.scriptId)) {
throw new BetaError(`script ${s.scriptId} listed twice in one beta`, 'duplicate-script');
}
seenScriptIds.add(s.scriptId);
const script = getScript(s.scriptId);
if (!script) throw new BetaError(`no such script (id=${s.scriptId})`, 'no-such-script');
const version = getVersion(s.scriptVersionId);
if (!version) throw new BetaError(`no such version (id=${s.scriptVersionId})`, 'no-such-version');
if (version.script_id !== script.id) {
throw new BetaError(
`version ${s.scriptVersionId} does not belong to script ${s.scriptId}`,
'version-script-mismatch'
);
}
if (!isCollapsingKind(script.kind)) {
const targetKey = `${script.kind}:${script.target_rel_path}`;
if (seenTargets.has(targetKey)) {
throw new BetaError(
`${script.kind}s ${seenTargets.get(targetKey)} and ${script.id} share target ${script.target_rel_path}`,
'duplicate-target'
);
}
seenTargets.set(targetKey, script.id);
}
if (script.kind === 'baseline') {
if (baselineInBeta) {
throw new BetaError(
`two baselines in one beta (${baselineInBeta.script.id}, ${script.id}) — betas support at most one baseline`,
'duplicate-target'
);
}
baselineInBeta = { script, version };
}
resolved.push({ script, version });
}
// Look up server rows once.
const serverRows: Array<{ name: string; game: string | null; map: string | null }> = [];
for (const name of input.servers) {
const row = getServer(name);
if (!row) throw new BetaError(`unknown server: ${name}`, 'unknown-server');
serverRows.push({ name, game: row.game, map: row.map });
}
if (plan && ['map', 'game', 'map-game'].includes(plan.scope)) {
const missing = serverRows.some(s => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map));
if (missing) throw new BetaError('Coverage requires game/map metadata on every selected server', 'invalid-state');
}
// Per-script scope validation. Aggregate all violations into one error.
const outOfScope: Array<{ scriptId: number; scope: string; servers: string[] }> = [];
for (const { script } of resolved) {
if (!script.scope_type || !script.scope_value) continue;
const bad = serverRows.filter(
(r) => (script.scope_type === 'game' ? r.game : r.map) !== script.scope_value
);
if (bad.length > 0) {
outOfScope.push({
scriptId: script.id,
scope: `${script.scope_type}=${script.scope_value}`,
servers: bad.map((r) => r.name)
});
}
}
if (outOfScope.length > 0) {
throw new BetaError('one or more servers are out of scope', 'out-of-scope', outOfScope);
}
// A beta outranks campaigns during activation. Do not let it overtake older
// staged writes which would otherwise overwrite the tested version on closure.
// Legacy campaigns have no staged payload and retain their restart-only priority.
const queuedCampaigns = input.servers.flatMap(unresolvedCampaignWrites);
if (queuedCampaigns.length) {
const interrupted = queuedCampaigns.some(row => row.state === 'applying');
throw new BetaError(interrupted
? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.'
: 'Finish or resume queued file campaigns before opening a beta on these servers.', 'conflict', queuedCampaigns);
}
// Per-(server, script) conflict check. heldByBeta covers "another running
// beta already claimed this target"; the previous stale-.OLD check is no
// longer relevant now that snapshots live in the DB.
const conflicts: Array<{ server: string; scriptId: number; heldByBetaId: number }> = [];
for (const name of input.servers) {
const held = heldByBeta(name);
for (const { script } of resolved) {
const owner = held.get(script.target_rel_path);
if (owner) conflicts.push({ server: name, scriptId: script.id, heldByBetaId: owner });
}
}
if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts);
// DB writes in one transaction.
const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols
const insertBeta = db().prepare(
`INSERT INTO betas (script_id, script_version_id, trigger_kind, wait_for_health, wait_for_match, state, notes)
VALUES (?, ?, 'bundle', ?, ?, 'running', ?)`
);
const insertBetaScript = db().prepare(
`INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, ?)`
);
const insertBetaServer = db().prepare(
`INSERT INTO beta_servers (beta_id, server, had_original, prior_version_id, stage, updated_at)
VALUES (?, ?, 0, NULL, 'deploying', ?)`
);
const insertBetaServerScript = db().prepare(
`INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original, prior_version_id)
VALUES (?, ?, ?, ?, ?)`
);
let betaId!: number;
db().transaction(() => {
const res = insertBeta.run(
primary.script.id,
primary.version.id,
input.waitForHealth ? 1 : 0,
input.waitForMatch ? 1 : 0,
input.notes ?? null
);
betaId = Number(res.lastInsertRowid);
if (plan) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(plan));
for (const r of resolved) {
insertBetaScript.run(betaId, r.script.id, r.version.id, r.script.kind);
}
for (const name of input.servers) {
insertBetaServer.run(betaId, name, nowIso());
if (plan) {
const selected = serverRows.find(s => s.name === name)!;
db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map);
}
for (const r of resolved) {
insertBetaServerScript.run(
betaId,
name,
r.script.id,
0,
null
);
}
}
})();
if (plan?.waveSize) db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId);
await deployBetaServers(betaId, plan?.waveSize ? input.servers.slice(0, plan.waveSize) : input.servers);
emit({
type: 'beta-opened',
betaId,
scriptIds: resolved.map((r) => r.script.id),
servers: input.servers
});
return {
betaId,
servers: input.servers,
scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id }))
};
}
/** Apply a reserved wave only when its predecessors have passed. */
export async function deployBetaServers(betaId: number, names: string[]): Promise<void> {
return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
}
async function deployBetaServersInternal(betaId: number, names: string[]): Promise<void> {
for (const name of names) {
if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server');
}
const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! }));
const serverRows = names.map(name => getServer(name)!);
// Split resolved artifacts into non-collapsing (each writes its own file)
// and collapsing (config/baseline overlay onto one shared dedicated_sp.cfg).
const nonCollapsing = resolved.filter((r) => !isCollapsingKind(r.script.kind));
const collapsingOverlays = resolved.filter((r) => r.script.kind === 'config');
const collapsingBaseline = resolved.find((r) => r.script.kind === 'baseline');
// Archive: one copy per non-collapsing artifact under its target basename,
// plus one composed preview per beta server for the config layer.
const archive = betaArchiveOf(betaId);
await mkdir(archive, { recursive: true });
for (const r of nonCollapsing) {
const base = posix.basename(r.script.target_rel_path);
await writeFile(join(archive, base), r.version.content);
}
// Persist the requested bytes without touching any live destination. Provisioning
// captures the original file and inventory immediately before guarded activation.
for (const name of names) {
if (!isActiveBetaServer(betaId, name)) continue;
const serverRow = serverRows.find((r) => r.name === name)!;
try {
const changes: Parameters<typeof queueFileChange>[0][] = nonCollapsing.map(({ script, version }) => ({
sourceKind: 'beta' as const, sourceId: String(betaId), server: name,
kind: script.kind, relPath: script.target_rel_path, content: version.content,
scriptVersionId: version.id, inventorySource: `beta:${betaId}`
}));
if (collapsingBaseline || collapsingOverlays.length > 0) {
const composedResult = composeCfgForServer(name, serverRow.game, serverRow.map, {
inBetaBaseline: collapsingBaseline?.version,
inBetaOverlays: collapsingOverlays
});
if (!composedResult) {
throw new Error(`no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays`);
}
const composedBuf = Buffer.from(composedResult.text, 'utf8');
changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name,
kind: 'config', relPath: configFileOf(name), content: composedBuf,
scriptVersionId: null, inventorySource: `beta:${betaId}` });
await writeFile(join(archive, `${name}.${configFileOf(name)}`), composedBuf);
}
if (!isActiveBetaServer(betaId, name)) continue;
db().transaction(() => {
for (const change of changes) queueFileChange(change);
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL")
.run(nowIso(), betaId, name);
})();
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null });
} catch (error) {
const message = `deploy: ${(error as Error).message}`;
db().prepare("UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL")
.run(message, nowIso(), betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: message });
}
}
}
// ---------- rollback ----------
export interface RollbackResult {
betaId: number;
deploymentId: string | null;
restored: string[];
dropped: string[];
warnings: string[];
}
export async function rollbackBeta(betaId: number, options: { restart?: 'manual' | 'when-empty' } = {}): Promise<RollbackResult> {
return exclusive(betaId, () => rollbackBetaInternal(betaId, options));
}
async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise<RollbackResult> {
const beta = getBeta(betaId);
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
if (beta.state !== 'running' && beta.state !== 'ready-promote') {
throw new BetaError(`beta is ${beta.state}, cannot rollback`, 'invalid-state');
}
const bundleRows = listBetaScripts(betaId);
if (bundleRows.length === 0) {
throw new BetaError('beta has no scripts', 'no-scripts');
}
const bundle = bundleRows.map((br) => ({
script: getScript(br.script_id)!,
versionId: br.script_version_id
}));
const servers = listBetaServers(betaId);
const restored = new Set<string>();
const dropped = new Set<string>();
const warnings: string[] = [];
const hasCollapsing = bundle.some((b) => isCollapsingKind(b.script.kind));
const getSnapshot = db().prepare(
`SELECT prior_content, prior_sha256, prior_size FROM beta_file_snapshots
WHERE beta_id = ? AND server = ? AND rel_path = ?`
);
const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = [];
const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?');
const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => {
const staged = isFileApplied('beta', String(betaId), bs.server, kind, path);
if (staged !== null) return staged;
// Existing betas predating staging wrote their files before deployed_at.
return !!changed.get(betaId, bs.server, path) || !!bs.deployed_at;
};
for (const bs of servers) {
if (!isActiveBetaServer(betaId, bs.server)) {
warnings.push(`${bs.server}: original server was deleted; retained history was skipped`);
continue;
}
for (const { script } of bundle) {
if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) continue;
const bss = getBetaServerScript(betaId, bs.server, script.id);
const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as
| { prior_content: Buffer; prior_sha256: string; prior_size: number }
| undefined;
if (bss?.had_original && !snap) {
warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`);
continue;
}
restorations.push({ server: bs.server, kind: script.kind, relPath: script.target_rel_path,
content: snap?.prior_content ?? null, versionId: bss?.prior_version_id ?? null });
(snap ? restored : dropped).add(bs.server);
}
const targetRel = configFileOf(bs.server);
if (hasCollapsing && applied(bs, 'config', targetRel)) {
const snap = getSnapshot.get(betaId, bs.server, targetRel) as
| { prior_content: Buffer; prior_sha256: string; prior_size: number }
| undefined;
restorations.push({ server: bs.server, kind: 'config', relPath: targetRel,
content: snap?.prior_content ?? null, versionId: null });
(snap ? restored : dropped).add(bs.server);
}
}
const deploymentId = restorations.length ? `dep-beta-rollback-${betaId}-${Date.now()}` : null;
db().transaction(() => {
cancelStagedFiles('beta', String(betaId));
if (deploymentId) {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)")
.run(deploymentId, nowIso(), storedPlan(betaId) ? 'beta-test-plan' : 'ui', betaId);
for (const server of new Set(restorations.map(change => change.server))) {
db().prepare('INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)')
.run(deploymentId, server, restorations.filter(change => change.server === server).length,
options.restart === 'manual' ? 'held' : 'pending');
}
for (const change of restorations) {
const file = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server,
change.content === null ? 'removed' : 'updated', change.relPath,
change.content === null ? null : sha256Buffer(change.content), change.content);
queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server,
kind: change.kind, relPath: change.relPath, content: change.content,
scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`,
deploymentFileId: Number(file.lastInsertRowid) });
}
}
// The queued restoration owns a durable copy before snapshots are released.
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId);
})();
if (deploymentId) emit({ type: 'deployment-opened', deploymentId, action: 'rollback', servers: [...new Set(restorations.map(change => change.server))] });
// Wipe archive + close out beta.
const archive = betaArchiveOf(betaId);
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
// Per-version test-failed transition. Skip a version if (a) another beta
// still gates it at ready-promote/promoted, or (b) it's the currently-
// promoted version of its script (already-live code isn't "failed" by
// rejecting a redundant beta).
for (const { script, versionId } of bundle) {
const stillOk = db()
.prepare(
`SELECT 1 FROM beta_scripts bs
JOIN betas b ON b.id = bs.beta_id
WHERE bs.script_version_id = ?
AND b.id <> ?
AND b.state IN ('ready-promote','promoted')
LIMIT 1`
)
.get(versionId, betaId);
const isCurrent = script.current_version_id === versionId;
if (!stillOk && !isCurrent) {
db().prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`).run(versionId);
}
}
emit({ type: 'beta-state', betaId, state: 'rolled-back' });
return {
betaId,
deploymentId,
restored: [...restored].sort(),
dropped: [...dropped].sort(),
warnings
};
}
// ---------- promote ----------
export interface PromoteResult {
betaId: number;
deploymentId: string;
landed: string[]; // unique servers that received at least one file
scriptIds: number[];
awaitingActivation: number;
failedServers: string[];
}
/** Snapshot exactly the scope expansion and composed configuration reviewed by the operator. */
function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configContent: Map<string, Buffer> } {
const beta = getBeta(betaId);
if (!beta) throw new BetaError('This beta no longer exists.', 'no-such-beta');
const bundle = listBetaScripts(betaId).map(row => {
const script = getScript(row.script_id), version = getVersion(row.script_version_id);
if (!script || !version) throw new BetaError('A selected file version is no longer available.', 'no-such-version');
return { script, version };
});
if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts');
const participants = listBetaServers(betaId);
const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server));
const targets = (script: ScriptRow) => [...new Set([...tested, ...(script.scope_type && script.scope_value
? serversMatchingScope(script.scope_type, script.scope_value) : [])])].sort();
const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind,
path: script.target_rel_path, versionId: version.id, version: version.version,
release_version: version.release_version, sha256: version.sha256,
scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) }));
const names = [...new Set(files.flatMap(file => file.servers))].sort();
const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server))
.map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`);
const servers = names.map(name => {
const server = getServer(name);
if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`);
return { name, tested: tested.has(name), game: server?.game ?? null, map: server?.map ?? null };
});
const collapsing = bundle.filter(({ script }) => isCollapsingKind(script.kind));
const configContent = new Map<string, Buffer>();
const configWrites = [...new Set(collapsing.flatMap(({ script }) => targets(script)))].sort().map(server => {
const row = getServer(server);
const composed = row && composeCfgForServer(server, row.game, row.map, {
inBetaBaseline: collapsing.find(({ script }) => script.kind === 'baseline')?.version,
inBetaOverlays: collapsing.filter(({ script }) => script.kind === 'config')
});
const error = composed ? null : `${server}: no baseline is available to compose its configuration.`;
if (error) blockers.push(error);
const content = composed ? Buffer.from(composed.text, 'utf8') : null;
if (content) configContent.set(server, content);
return { server, path: configFileOf(server), sha256: content ? sha256Buffer(content) : null, error };
});
const impact = { betaId, files, servers, configWrites, blockers, restartPolicy: 'additional-when-empty' as const };
return { preview: { ...impact, token: sha256Buffer(Buffer.from(JSON.stringify(impact))), createdAt: nowIso() }, configContent };
}
export function previewBetaPromotion(betaId: number): BetaPromotionPreview {
return promotionPlan(betaId).preview;
}
function promoteDeploymentId(betaId: number): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-promote-${betaId}`;
}
export async function promoteBeta(betaId: number, reviewedToken?: string): Promise<PromoteResult> {
if (storedPlan(betaId)) {
const { plannedBetaTick } = await import('../pipeline/beta-plan-runner.js');
await plannedBetaTick();
}
return exclusive(betaId, () => promoteBetaInternal(betaId, reviewedToken));
}
async function promoteBetaInternal(betaId: number, reviewedToken?: string): Promise<PromoteResult> {
const test = storedPlan(betaId);
if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote');
const beta = getBeta(betaId);
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
if (beta.state !== 'ready-promote') {
throw new BetaError(`beta must be ready-promote (is ${beta.state})`, 'not-ready-promote');
}
const reviewed = promotionPlan(betaId);
const checkReview = () => {
const current = previewBetaPromotion(betaId);
if (current.token !== (reviewedToken ?? reviewed.preview.token)) {
throw new BetaError('Promotion targets or versions changed. Review the updated plan before promoting.', 'preview-changed', current);
}
if (current.blockers.length) throw new BetaError(current.blockers.join(' '), 'invalid-state');
};
checkReview();
const bundleRows = listBetaScripts(betaId);
if (bundleRows.length === 0) throw new BetaError('beta has no scripts', 'no-scripts');
const bundle = bundleRows.map((br) => ({
script: getScript(br.script_id)!,
version: getVersion(br.script_version_id)!
}));
const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server));
// Split bundle into non-collapsing (per-file) and collapsing (composed cfg).
const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind));
const collapsing = bundle.filter((b) => isCollapsingKind(b.script.kind));
// Build a per-(script, server) plan for non-collapsing artifacts.
interface Plan {
script: ScriptRow;
version: ScriptVersionRow;
server: string;
priorVersionId: number | null;
priorSha: string | null;
priorContent: Buffer | null;
hadFile: boolean;
}
const plans: Plan[] = [];
const getSnapshotForPromote = db().prepare(
`SELECT prior_content, prior_sha256 FROM beta_file_snapshots
WHERE beta_id = ? AND server = ? AND rel_path = ?`
);
for (const { script, version } of nonCollapsing) {
const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers;
for (const name of targets) {
const dest = destOfKind(name, script.kind);
let plan: Plan;
if (betaServerSet.has(name)) {
const bss = getBetaServerScript(betaId, name, script.id);
if (bss?.had_original) {
const snap = getSnapshotForPromote.get(betaId, name, script.target_rel_path) as
| { prior_content: Buffer; prior_sha256: string }
| undefined;
let priorContent: Buffer | null = null;
let priorSha: string | null = null;
if (snap) {
priorContent = snap.prior_content;
priorSha = snap.prior_sha256;
} else if (bss.prior_version_id) {
const pv = getVersion(bss.prior_version_id);
if (pv) { priorContent = pv.content; priorSha = pv.sha256; }
}
plan = {
script, version, server: name,
priorVersionId: bss.prior_version_id, priorSha, priorContent, hadFile: true
};
} else {
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
} else {
// Non-beta matching-scope server: current on-disk file (if any)
// is the "before" state for a promote-then-rollback round trip.
const targetPath = join(dest, script.target_rel_path);
if (existsSync(targetPath)) {
const priorContent = await readFile(targetPath);
const inv = getServerScript(name, script.target_rel_path);
plan = {
script, version, server: name,
priorVersionId: inv?.script_version_id ?? null,
priorSha: sha256Buffer(priorContent),
priorContent, hadFile: true
};
} else {
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
}
plans.push(plan);
}
}
// Build durable activation work; live inventory changes only at activation.
const serverErr = new Map<string, string>();
// Collapsing config layer: one composed cfg per server. Union of beta
// servers + scope-matching servers for every collapsing artifact.
interface CfgPlan {
server: string;
priorContent: Buffer | null;
priorSha: string | null;
composedContent: Buffer;
composedSha: string;
hadFile: boolean;
requiresActivation: boolean;
}
const cfgPlans: CfgPlan[] = [];
if (collapsing.length > 0) {
const cfgTargets = reviewed.preview.configWrites.map(write => write.server);
for (const name of cfgTargets) {
const srv = getServer(name);
if (!srv) {
serverErr.set(name, 'unknown server during config promote');
continue;
}
const composedBuf = reviewed.configContent.get(name);
if (!composedBuf) {
serverErr.set(name, `no baseline available for ${name} (game=${srv.game ?? '?'})`);
continue;
}
const composedSha = sha256Buffer(composedBuf);
const dest = destOfKind(name, 'config');
const targetRel = configFileOf(name);
const targetPath = join(dest, targetRel);
let priorContent: Buffer | null = null;
let priorSha: string | null = null;
let hadFile = false;
// Beta server: preserve its original bytes captured at activation.
if (betaServerSet.has(name)) {
const snap = getSnapshotForPromote.get(betaId, name, targetRel) as
| { prior_content: Buffer; prior_sha256: string }
| undefined;
if (snap) {
priorContent = snap.prior_content;
priorSha = snap.prior_sha256;
hadFile = true;
}
}
if (!betaServerSet.has(name) && existsSync(targetPath)) {
// Capture an initial preview; activation records the actual before image.
priorContent = await readFile(targetPath);
priorSha = sha256Buffer(priorContent);
hadFile = true;
}
const requiresActivation = !betaServerSet.has(name) || !existsSync(targetPath)
|| sha256Buffer(await readFile(targetPath)) !== composedSha;
cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation });
}
}
// Recheck after asynchronous snapshot reads, before any mutation. Execution uses
// the captured target lists/content so later membership changes never expand it.
checkReview();
// A scope expansion from another promotion may have been queued while this
// beta was active. Reassert the tested bytes after that older campaign, even
// when they already match the live file, so it cannot overwrite this release.
const queuedByServer = new Map(reviewed.preview.servers.map(server => [server.name, unresolvedCampaignWrites(server.name)]));
const plansToActivate = new Set(plans.filter(p => !betaServerSet.has(p.server)
|| queuedByServer.get(p.server)?.some(row => row.kind === p.script.kind && row.rel_path === p.script.target_rel_path)));
for (const cp of cfgPlans) {
if (queuedByServer.get(cp.server)?.some(row => row.kind === 'config' && row.rel_path === configFileOf(cp.server))) cp.requiresActivation = true;
}
const activationServers = new Set([...Array.from(plansToActivate, plan => plan.server),
...cfgPlans.filter(plan => plan.requiresActivation).map(plan => plan.server)]);
// Build the aggregate deployment.
const depId = promoteDeploymentId(betaId);
const startedAt = nowIso();
const insertDep = db().prepare(
`INSERT INTO deployments
(id, action, started_at, triggered_by, script_id, script_version_id, prior_version_id, beta_id)
VALUES (?, 'promote', ?, 'ui', NULL, NULL, NULL, ?)`
);
const insertDepScript = db().prepare(
`INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind)
VALUES (?, ?, ?, ?, ?)`
);
const insertDepServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
VALUES (?, ?, ?, ?, NULL, NULL, ?, 0, ?)`
);
const insertDepFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
);
const uniqueServers = Array.from(new Set([
...plans.map((p) => p.server),
...cfgPlans.map((c) => c.server)
]));
db().transaction(() => {
insertDep.run(depId, startedAt, betaId);
for (const { script, version } of bundle) {
insertDepScript.run(depId, script.id, version.id, script.current_version_id, script.kind);
}
// Record the reviewed change. Scope expansion gets its actual before
// image when activated; tested servers keep their pre-beta history.
for (const p of plans) {
const verb = p.hadFile ? 'updated' : 'added';
const file = insertDepFile.run(
depId,
p.server,
verb,
p.script.target_rel_path,
p.priorSha,
p.version.sha256,
p.priorContent,
p.version.content
);
if (plansToActivate.has(p)) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server,
kind: p.script.kind, relPath: p.script.target_rel_path, content: p.version.content,
scriptVersionId: p.version.id, inventorySource: `promote:${depId}`,
deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) });
}
}
for (const cp of cfgPlans) {
const verb = cp.hadFile ? 'updated' : 'added';
const file = insertDepFile.run(
depId,
cp.server,
verb,
configFileOf(cp.server),
cp.priorSha,
cp.composedSha,
cp.priorContent,
cp.composedContent
);
if (cp.requiresActivation) {
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server,
kind: 'config', relPath: configFileOf(cp.server), content: cp.composedContent,
scriptVersionId: null, inventorySource: `promote:${depId}`,
deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(cp.server) });
}
}
// deployment_servers — one row per unique server. Beta servers are
// already restarted; scope-only servers still need the sweep to drain
// and restart them.
for (const name of uniqueServers) {
const err = serverErr.get(name);
if (err) {
insertDepServer.run(depId, name, 0, 'failed', null, err);
continue;
}
const filesChanged =
plans.filter((p) => p.server === name).length +
cfgPlans.filter((c) => c.server === name).length;
if (!activationServers.has(name)) {
const bsRestartedAt = db()
.prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`)
.get(betaId, name) as { restarted_at: string | null } | undefined;
insertDepServer.run(depId, name, filesChanged, 'done', bsRestartedAt?.restarted_at ?? startedAt, null);
} else {
insertDepServer.run(depId, name, filesChanged, 'pending', null, null);
}
}
// Update each script's current_version_id in the same transaction.
for (const { script, version } of bundle) {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
}
})();
db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId);
// The deployment history owns the pre-beta bytes for future rollback.
db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
const archive = betaArchiveOf(betaId);
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
const landed = Array.from(new Set([
...plans.map((p) => p.server),
...cfgPlans.map((c) => c.server)
])).sort();
emit({ type: 'beta-state', betaId, state: 'promoted' });
emit({ type: 'deployment-opened', deploymentId: depId, action: 'promote', servers: landed });
return {
betaId,
deploymentId: depId,
landed,
scriptIds: bundle.map((b) => b.script.id),
awaitingActivation: [...activationServers].filter(name => !serverErr.has(name)).length,
failedServers: [...serverErr.keys()]
};
}
// ---------- helpers used by API layer ----------
// Latest of the script's versions — useful for the "open beta with this
// version" default in the UI.
export function latestVersionForScript(script: ScriptRow): ScriptVersionRow | null {
const rows = listVersions(script.id);
return rows[0] ?? null;
}
/**
* Un-set a script's scope. Files are left on disk so drift picks up the
* change on the next /deployments rescan and the operator reconciles from
* there like any other drift — the same visible flow as manual drift.
*
* For scripts/plugins, we detach the inventory rows for this artifact's
* versions so drift Pass 1 flags the on-disk copies as untracked. For
* configs/baselines, no detach is needed: drift Pass 2 recomposes the
* expected dedicated_sp.cfg against the current promoted overlays, and
* scope=null naturally drops the overlay from the compose, surfacing the
* disparity. Returns the servers whose inventory was detached (empty for
* configs/baselines).
*/
export function unsetScopeAndDetach(scriptId: number): string[] {
const script = getScript(scriptId);
if (!script) throw new BetaError(`no such script (id=${scriptId})`, 'no-such-script');
setScope({ scriptId, scopeType: null, scopeValue: null });
const detached: string[] = [];
if (script.kind === 'script' || script.kind === 'plugin') {
const rows = db()
.prepare(
`SELECT server FROM server_scripts
WHERE rel_path = ?
AND script_version_id IN (SELECT id FROM script_versions WHERE script_id = ?)`
)
.all(script.target_rel_path, scriptId) as Array<{ server: string }>;
for (const { server } of rows) {
removeScript(server, script.target_rel_path);
detached.push(server);
}
}
return detached;
}