import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { storedPlan } from './beta-tests.js'; import type { BetaPromotionPreview } from '../../beta-promotion.js'; const operations = new Set(); export const betaOperationInFlight = (id: number) => operations.has(id); async function exclusive(id: number, action: () => Promise): Promise { if (operations.has(id)) throw new BetaError('A beta operation is already running', 'invalid-state'); operations.add(id); try { return await action(); } finally { operations.delete(id); } } import { mkdir, rm, writeFile } from 'node:fs/promises'; import { readFile, exists } from '../backends/files.js'; import { getHostOperation } from '../hosts/index.js'; import { ensureHostSchema } from '../hosts/schema.js'; import { join, posix } from 'node:path'; import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { sha256Buffer } from './files.js'; import { getServer, heldByBeta } from './servers.js'; import { getBaselineForScope, getScript, getVersion, listPromotedOverlaysForScope, listVersions, serversMatchingScope, setScope, type ScopeType, type ScriptRow, type ScriptVersionRow } from './scripts.js'; import { composeServerCfg } from './config-compose.js'; import { getServerScript, removeScript } from './inventory.js'; import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js'; import { emit } from '../pipeline/bus.js'; import { cancelProvisioningJob } from '../provisioning/cancellation.js'; import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js'; import { requestRestart } from '../provisioning/requests.js'; import type { BetaStage } from '../pipeline/bus.js'; // Whether an artifact writes a dedicated on-disk file (script/plugin) or // contributes to a composed shared file (config/baseline). function isCollapsingKind(kind: ArtifactKind): boolean { return kind === 'config' || kind === 'baseline'; } // The path we should write to for the composed cfg on this server. function composedCfgPath(server: string): string { return join(destOfKind(server, 'config'), configFileOf(server)); } // Compose the effective dedicated_sp.cfg for a given beta+server, taking // into account: // • baseline: from the beta if present, else the currently-promoted // baseline for the server's scope // • promoted overlays for the server's scope (excluding scripts being // replaced by this beta) // • this beta's overlays whose scope matches the server // // Returns null if no baseline is available (nothing to compose against). interface ComposeInputs { inBetaBaseline?: ScriptVersionRow; inBetaOverlays: Array<{ script: ScriptRow; version: ScriptVersionRow }>; } function scopeAppliesToServer( scope: { scope_type: string | null; scope_value: string | null }, server: { game: string | null; map: string | null } ): boolean { if (!scope.scope_type || !scope.scope_value) return false; const field = scope.scope_type === 'game' ? server.game : server.map; return field === scope.scope_value; } function composeCfgForServer( serverName: string, serverGame: string | null, serverMap: string | null, inputs: ComposeInputs ): { text: string; baselineScriptId: number | null } | null { // Baseline: prefer the beta's baseline if its scope matches this server. let baselineText: string | null = null; let baselineScriptId: number | null = null; if (inputs.inBetaBaseline) { const s = getScript(inputs.inBetaBaseline.script_id); if (s && scopeAppliesToServer(s, { game: serverGame, map: serverMap })) { baselineText = inputs.inBetaBaseline.content.toString('utf8'); baselineScriptId = s.id; } } // Fall back to the currently-promoted baseline for the server's game. if (baselineText === null && serverGame) { const promoted = getBaselineForScope('game', serverGame); if (promoted?.current_version_id) { const ver = getVersion(promoted.current_version_id); if (ver) { baselineText = ver.content.toString('utf8'); baselineScriptId = promoted.id; } } } if (baselineText === null) return null; // In-beta script ids that are replacing promoted overlays — those get // skipped from the promoted overlay list to avoid double-applying an // older version of the same script. const inBetaScriptIds = new Set(inputs.inBetaOverlays.map((o) => o.script.id)); // Collect promoted overlays scoped to this server's game AND map. const promotedOverlays: Array<{ id: number; text: string }> = []; const promotedIdsSeen = new Set(); const pushIfScoped = (rows: ScriptRow[]) => { for (const s of rows) { if (promotedIdsSeen.has(s.id)) continue; if (inBetaScriptIds.has(s.id)) continue; if (!scopeAppliesToServer(s, { game: serverGame, map: serverMap })) continue; if (!s.current_version_id) continue; const ver = getVersion(s.current_version_id); if (!ver) continue; promotedOverlays.push({ id: s.id, text: ver.content.toString('utf8') }); promotedIdsSeen.add(s.id); } }; if (serverGame) pushIfScoped(listPromotedOverlaysForScope('game', serverGame)); if (serverMap) pushIfScoped(listPromotedOverlaysForScope('map', serverMap)); // In-beta overlays scoped to this server. const inBetaScopedOverlays = inputs.inBetaOverlays .filter(({ script }) => scopeAppliesToServer(script, { game: serverGame, map: serverMap })) .map(({ script, version }) => ({ id: script.id, text: version.content.toString('utf8') })); const composed = composeServerCfg(baselineText, [...promotedOverlays, ...inBetaScopedOverlays], serverGame, serverMap, getServer(serverName)?.map_slug); return { text: composed, baselineScriptId }; } export interface BetaRow { id: number; // Deprecated: the columns are still populated (NOT NULL on legacy schemas) // from the first bundled script's identity so existing data continues to // roundtrip, but new code reads beta_scripts for the authoritative list. script_id: number; script_version_id: number; trigger_kind: string; wait_for_health: number; wait_for_match: number; state: 'running' | 'ready-promote' | 'promoted' | 'rolled-back'; opened_at: string; closed_at: string | null; notes: string | null; } export interface BetaServerRow { beta_id: number; server: string; server_deleted_at: string | null; had_original: number; // deprecated; per-script data lives in beta_server_scripts prior_version_id: number | null; // deprecated stage: BetaStage; error: string | null; deployed_at: string | null; restarted_at: string | null; provisioning_id: string | null; healthy_at: string | null; match_seen_at: string | null; updated_at: string; } export interface BetaScriptRow { beta_id: number; script_id: number; script_version_id: number; kind: ArtifactKind; } export interface BetaServerScriptRow { beta_id: number; server: string; script_id: number; had_original: number; prior_version_id: number | null; } export class BetaError extends Error { constructor( message: string, public readonly kind: | 'no-such-script' | 'no-such-version' | 'version-script-mismatch' | 'no-scripts' | 'no-servers' | 'unknown-server' | 'out-of-scope' | 'conflict' | 'blocked-backup' | 'no-such-beta' | 'not-ready-promote' | 'not-running' | 'dest-fail' | 'invalid-state' | 'duplicate-script' | 'preview-changed' | 'duplicate-target', public readonly details?: unknown ) { super(message); } } function assertNativeActivationSettled(betaId: number): void { const pending = db().prepare(`SELECT n.operation_id FROM native_provisioning n JOIN provisioning_sources s ON s.operation_id=n.deployment_id WHERE s.kind='beta' AND s.source_id=? AND n.operation_id IS NOT NULL AND n.files_json IS NOT NULL AND n.applied_at IS NULL`).all(String(betaId)) as { operation_id: string }[]; if (pending.some(row => getHostOperation(row.operation_id)?.state !== 'failed')) throw new BetaError('The host is still reconciling file activation. Wait for its durable before images before promoting or rolling back.', 'conflict'); } function nowIso(): string { return new Date().toISOString(); } function unresolvedCampaignWrites(server: string): Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }> { return db().prepare(`SELECT DISTINCT f.server, f.source_id AS deploymentId, f.state, f.kind, f.rel_path FROM staged_file_changes f JOIN deployment_servers ds ON ds.deployment_id = f.source_id AND ds.server = f.server LEFT JOIN server_deployments operation ON operation.id = ds.provisioning_id WHERE f.source_kind = 'campaign' AND f.server = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed','held','manual') AND (f.state = 'applying' OR (f.state = 'pending' AND (ds.restart_state IN ('held','manual') OR operation.id IS NULL OR operation.state IN ('queued','waiting','running','awaiting-health'))))`) .all(server) as Array<{ server: string; deploymentId: string; state: string; kind: ArtifactKind; rel_path: string }>; } // ---------- Reads ---------- export function getBeta(id: number): BetaRow | undefined { return db().prepare('SELECT * FROM betas WHERE id = ?').get(id) as BetaRow | undefined; } export function listBetas(state?: BetaRow['state']): BetaRow[] { if (state) { return db().prepare('SELECT * FROM betas WHERE state = ? ORDER BY opened_at DESC').all(state) as BetaRow[]; } return db().prepare('SELECT * FROM betas ORDER BY opened_at DESC').all() as BetaRow[]; } export function listBetaServers(betaId: number): BetaServerRow[] { return db() .prepare('SELECT * FROM beta_servers WHERE beta_id = ? ORDER BY server') .all(betaId) as BetaServerRow[]; } /** History retains names, so operational callers must also check the participant's identity. */ export function isActiveBetaServer(betaId: number, server: string): boolean { return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`) .get(betaId, server); } export function listBetaScripts(betaId: number): BetaScriptRow[] { return db() .prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id') .all(betaId) as BetaScriptRow[]; } export function getBetaServerScript(betaId: number, server: string, scriptId: number): BetaServerScriptRow | undefined { return db() .prepare('SELECT * FROM beta_server_scripts WHERE beta_id = ? AND server = ? AND script_id = ?') .get(betaId, server, scriptId) as BetaServerScriptRow | undefined; } // A script is "in a running beta" if beta_scripts joins to a running beta row. // Existing callers only care about the running-beta id, so return that shape. export function findRunningBetaForScript(scriptId: number): BetaRow | undefined { return db() .prepare( `SELECT b.* FROM betas b JOIN beta_scripts bs ON bs.beta_id = b.id WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(scriptId) as BetaRow | undefined; } // ---------- openBeta ---------- export interface OpenBetaScript { scriptId: number; scriptVersionId: number; } export interface OpenBetaInput { scripts: OpenBetaScript[]; servers: string[]; testPlan?: BetaPlan; waitForHealth: boolean; waitForMatch: boolean; notes?: string; } export interface OpenBetaResult { betaId: number; servers: string[]; scripts: OpenBetaScript[]; } export async function openBeta(input: OpenBetaInput): Promise { const plan = input.testPlan ? parseBetaPlan(input.testPlan) : null; if (new Set(input.servers).size !== input.servers.length) throw new BetaError("Select each server only once", "invalid-state"); if (plan?.scope === "quorum" && plan.quorum > input.servers.length) throw new BetaError("Quorum exceeds selected servers", "invalid-state"); if (!input.scripts || input.scripts.length === 0) { throw new BetaError('at least one script is required', 'no-scripts'); } if (!input.servers || input.servers.length === 0) { throw new BetaError('at least one server is required', 'no-servers'); } // Resolve every (script, version) pair and validate belonging. interface Resolved { script: ScriptRow; version: ScriptVersionRow } const resolved: Resolved[] = []; const seenScriptIds = new Set(); // For non-collapsing kinds (script/plugin), two artifacts targeting the // same rel_path within one beta is an internal race — reject up front. // Collapsing kinds (config/baseline) intentionally share a target file // (dedicated_sp.cfg) and compose, so they skip this check. const seenTargets = new Map(); let baselineInBeta: Resolved | null = null; for (const s of input.scripts) { if (seenScriptIds.has(s.scriptId)) { throw new BetaError(`script ${s.scriptId} listed twice in one beta`, 'duplicate-script'); } seenScriptIds.add(s.scriptId); const script = getScript(s.scriptId); if (!script) throw new BetaError(`no such script (id=${s.scriptId})`, 'no-such-script'); const version = getVersion(s.scriptVersionId); if (!version) throw new BetaError(`no such version (id=${s.scriptVersionId})`, 'no-such-version'); if (version.script_id !== script.id) { throw new BetaError( `version ${s.scriptVersionId} does not belong to script ${s.scriptId}`, 'version-script-mismatch' ); } if (!isCollapsingKind(script.kind)) { const targetKey = `${script.kind}:${script.target_rel_path}`; if (seenTargets.has(targetKey)) { throw new BetaError( `${script.kind}s ${seenTargets.get(targetKey)} and ${script.id} share target ${script.target_rel_path}`, 'duplicate-target' ); } seenTargets.set(targetKey, script.id); } if (script.kind === 'baseline') { if (baselineInBeta) { throw new BetaError( `two baselines in one beta (${baselineInBeta.script.id}, ${script.id}) — betas support at most one baseline`, 'duplicate-target' ); } baselineInBeta = { script, version }; } resolved.push({ script, version }); } // Look up server rows once. const serverRows: Array<{ name: string; game: string | null; map: string | null }> = []; for (const name of input.servers) { const row = getServer(name); if (!row) throw new BetaError(`unknown server: ${name}`, 'unknown-server'); // Once teardown is queued or in flight, opening a beta cannot safely // claim the instance. A failed remote request may still be running. const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health') OR (state IN ('failed', 'interrupted') AND stage IN ( 'Stopping and removing container', 'Removing server files and registry'))) LIMIT 1`).get(name); if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict'); serverRows.push({ name, game: row.game, map: row.map }); } if (plan && ['map', 'game', 'map-game'].includes(plan.scope)) { const missing = serverRows.some(s => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map)); if (missing) throw new BetaError('Coverage requires game/map metadata on every selected server', 'invalid-state'); } // Per-script scope validation. Aggregate all violations into one error. const outOfScope: Array<{ scriptId: number; scope: string; servers: string[] }> = []; for (const { script } of resolved) { if (!script.scope_type || !script.scope_value) continue; const bad = serverRows.filter( (r) => (script.scope_type === 'game' ? r.game : r.map) !== script.scope_value ); if (bad.length > 0) { outOfScope.push({ scriptId: script.id, scope: `${script.scope_type}=${script.scope_value}`, servers: bad.map((r) => r.name) }); } } if (outOfScope.length > 0) { throw new BetaError('one or more servers are out of scope', 'out-of-scope', outOfScope); } // A beta outranks campaigns during activation. Do not let it overtake older // staged writes which would otherwise overwrite the tested version on closure. // Legacy campaigns have no staged payload and retain their restart-only priority. const queuedCampaigns = input.servers.flatMap(unresolvedCampaignWrites); if (queuedCampaigns.length) { const interrupted = queuedCampaigns.some(row => row.state === 'applying'); throw new BetaError(interrupted ? 'A campaign file write needs inspection. Resolve or roll back that campaign before opening a beta on this server.' : 'Finish or resume queued file campaigns before opening a beta on these servers.', 'conflict', queuedCampaigns); } // Per-(server, script) conflict check. heldByBeta covers "another running // beta already claimed this target"; the previous stale-.OLD check is no // longer relevant now that snapshots live in the DB. const conflicts: Array<{ server: string; scriptId: number; heldByBetaId: number }> = []; for (const name of input.servers) { const held = heldByBeta(name); for (const { script } of resolved) { const owner = held.get(script.target_rel_path); if (owner) conflicts.push({ server: name, scriptId: script.id, heldByBetaId: owner }); } } if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts); // DB writes in one transaction. const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols const insertBeta = db().prepare( `INSERT INTO betas (script_id, script_version_id, trigger_kind, wait_for_health, wait_for_match, state, notes) VALUES (?, ?, 'bundle', ?, ?, 'running', ?)` ); const insertBetaScript = db().prepare( `INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, ?)` ); const insertBetaServer = db().prepare( `INSERT INTO beta_servers (beta_id, server, had_original, prior_version_id, stage, updated_at) VALUES (?, ?, 0, NULL, 'deploying', ?)` ); const insertBetaServerScript = db().prepare( `INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original, prior_version_id) VALUES (?, ?, ?, ?, ?)` ); let betaId!: number; db().transaction(() => { const res = insertBeta.run( primary.script.id, primary.version.id, input.waitForHealth ? 1 : 0, input.waitForMatch ? 1 : 0, input.notes ?? null ); betaId = Number(res.lastInsertRowid); if (plan) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(plan)); for (const r of resolved) { insertBetaScript.run(betaId, r.script.id, r.version.id, r.script.kind); } for (const name of input.servers) { insertBetaServer.run(betaId, name, nowIso()); if (plan) { const selected = serverRows.find(s => s.name === name)!; db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map); } for (const r of resolved) { insertBetaServerScript.run( betaId, name, r.script.id, 0, null ); } } })(); if (plan?.waveSize) db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId); await deployBetaServers(betaId, plan?.waveSize ? input.servers.slice(0, plan.waveSize) : input.servers); emit({ type: 'beta-opened', betaId, scriptIds: resolved.map((r) => r.script.id), servers: input.servers }); return { betaId, servers: input.servers, scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id })) }; } /** Apply a reserved wave only when its predecessors have passed. */ export async function deployBetaServers(betaId: number, names: string[]): Promise { return exclusive(betaId, () => deployBetaServersInternal(betaId, names)); } async function deployBetaServersInternal(betaId: number, names: string[]): Promise { for (const name of names) { if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server'); } const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! })); const serverRows = names.map(name => getServer(name)!); // Split resolved artifacts into non-collapsing (each writes its own file) // and collapsing (config/baseline overlay onto one shared dedicated_sp.cfg). const nonCollapsing = resolved.filter((r) => !isCollapsingKind(r.script.kind)); const collapsingOverlays = resolved.filter((r) => r.script.kind === 'config'); const collapsingBaseline = resolved.find((r) => r.script.kind === 'baseline'); // Archive: one copy per non-collapsing artifact under its target basename, // plus one composed preview per beta server for the config layer. const archive = betaArchiveOf(betaId); await mkdir(archive, { recursive: true }); for (const r of nonCollapsing) { const base = posix.basename(r.script.target_rel_path); await writeFile(join(archive, base), r.version.content); } // Persist the requested bytes without touching any live destination. Provisioning // captures the original file and inventory immediately before guarded activation. for (const name of names) { if (!isActiveBetaServer(betaId, name)) continue; const serverRow = serverRows.find((r) => r.name === name)!; try { const changes: Parameters[0][] = nonCollapsing.map(({ script, version }) => ({ sourceKind: 'beta' as const, sourceId: String(betaId), server: name, kind: script.kind, relPath: script.target_rel_path, content: version.content, scriptVersionId: version.id, inventorySource: `beta:${betaId}` })); if (collapsingBaseline || collapsingOverlays.length > 0) { const composedResult = composeCfgForServer(name, serverRow.game, serverRow.map, { inBetaBaseline: collapsingBaseline?.version, inBetaOverlays: collapsingOverlays }); if (!composedResult) { throw new Error(`no baseline available for ${name} (game=${serverRow.game ?? '?'}) — upload one before running config overlays`); } const composedBuf = Buffer.from(composedResult.text, 'utf8'); changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: 'config', relPath: configFileOf(name), content: composedBuf, scriptVersionId: null, inventorySource: `beta:${betaId}` }); await writeFile(join(archive, `${name}.${configFileOf(name)}`), composedBuf); } if (!isActiveBetaServer(betaId, name)) continue; db().transaction(() => { for (const change of changes) queueFileChange(change); db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, deployed_at = NULL, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") .run(nowIso(), betaId, name); })(); emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null }); } catch (error) { const message = `deploy: ${(error as Error).message}`; db().prepare("UPDATE beta_servers SET stage = 'failed', error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL") .run(message, nowIso(), betaId, name); emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: message }); } } } // ---------- rollback ---------- export interface RollbackResult { betaId: number; deploymentId: string | null; restored: string[]; dropped: string[]; warnings: string[]; } export async function rollbackBeta(betaId: number, options: { restart?: 'manual' | 'when-empty' } = {}): Promise { return exclusive(betaId, () => rollbackBetaInternal(betaId, options)); } async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise { const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'running' && beta.state !== 'ready-promote') { throw new BetaError(`beta is ${beta.state}, cannot rollback`, 'invalid-state'); } const bundleRows = listBetaScripts(betaId); if (bundleRows.length === 0) { throw new BetaError('beta has no scripts', 'no-scripts'); } const bundle = bundleRows.map((br) => ({ script: getScript(br.script_id)!, versionId: br.script_version_id })); ensureHostSchema(db()); db().transaction(() => { const jobs = db().prepare(`SELECT s.operation_id FROM provisioning_sources s WHERE s.kind='beta' AND s.source_id=?`) .all(String(betaId)) as { operation_id: string }[]; for (const job of jobs) { const undelivered = db().prepare(`SELECT 1 FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=? AND o.state='pending' AND o.last_delivered_at IS NULL`).get(job.operation_id); if (undelivered) cancelProvisioningJob(job.operation_id, `Superseded by rollback of beta #${betaId}`); reconcileNativeProvisioningReceipt(job.operation_id); } // Before images remain mandatory for a mutation already delivered to a host. // Completed commands can be reconciled here without waiting for game health. assertNativeActivationSettled(betaId); })(); const servers = listBetaServers(betaId); const restored = new Set(); const dropped = new Set(); const warnings: string[] = []; const hasCollapsing = bundle.some((b) => isCollapsingKind(b.script.kind)); const getSnapshot = db().prepare( `SELECT prior_content, prior_sha256, prior_size FROM beta_file_snapshots WHERE beta_id = ? AND server = ? AND rel_path = ?` ); const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = []; const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'); const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => { const staged = isFileApplied('beta', String(betaId), bs.server, kind, path); if (staged !== null) return staged; // Existing betas predating staging wrote their files before deployed_at. return !!changed.get(betaId, bs.server, path) || !!bs.deployed_at; }; for (const bs of servers) { if (!isActiveBetaServer(betaId, bs.server)) { warnings.push(`${bs.server}: original server was deleted; retained history was skipped`); continue; } for (const { script } of bundle) { if (isCollapsingKind(script.kind) || !applied(bs, script.kind, script.target_rel_path)) continue; const bss = getBetaServerScript(betaId, bs.server, script.id); const snap = getSnapshot.get(betaId, bs.server, script.target_rel_path) as | { prior_content: Buffer; prior_sha256: string; prior_size: number } | undefined; if (bss?.had_original && !snap) { warnings.push(`${bs.server}/${script.name}: snapshot missing — cannot restore`); continue; } restorations.push({ server: bs.server, kind: script.kind, relPath: script.target_rel_path, content: snap?.prior_content ?? null, versionId: bss?.prior_version_id ?? null }); (snap ? restored : dropped).add(bs.server); } const targetRel = configFileOf(bs.server); if (hasCollapsing && applied(bs, 'config', targetRel)) { const snap = getSnapshot.get(betaId, bs.server, targetRel) as | { prior_content: Buffer; prior_sha256: string; prior_size: number } | undefined; restorations.push({ server: bs.server, kind: 'config', relPath: targetRel, content: snap?.prior_content ?? null, versionId: null }); (snap ? restored : dropped).add(bs.server); } } const deploymentId = restorations.length ? `dep-beta-rollback-${betaId}-${Date.now()}` : null; db().transaction(() => { for (const bs of servers) { if (!isActiveBetaServer(betaId, bs.server)) continue; const pending = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).all(bs.server) as { id: string }[]; for (const job of pending) cancelProvisioningJob(job.id, `Superseded by rollback of beta #${betaId}`); } cancelStagedFiles('beta', String(betaId)); if (deploymentId) { db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)") .run(deploymentId, nowIso(), storedPlan(betaId) ? 'beta-test-plan' : 'ui', betaId); for (const server of new Set(restorations.map(change => change.server))) { db().prepare('INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, ?, ?)') .run(deploymentId, server, restorations.filter(change => change.server === server).length, options.restart === 'manual' ? 'held' : 'pending'); } for (const change of restorations) { const file = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) VALUES (?, ?, ?, ?, NULL, ?, NULL, ?)`).run(deploymentId, change.server, change.content === null ? 'removed' : 'updated', change.relPath, change.content === null ? null : sha256Buffer(change.content), change.content); queueFileChange({ sourceKind: 'campaign', sourceId: deploymentId, server: change.server, kind: change.kind, relPath: change.relPath, content: change.content, scriptVersionId: change.versionId, inventorySource: `rollback:${betaId}`, deploymentFileId: Number(file.lastInsertRowid) }); } } if (deploymentId) { for (const server of new Set(restorations.map(change => change.server))) { requestRestart(server, { kind: 'campaign', id: deploymentId }); } } // The queued restoration owns a durable copy before snapshots are released. db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId); })(); if (deploymentId) emit({ type: 'deployment-opened', deploymentId, action: 'rollback', servers: [...new Set(restorations.map(change => change.server))] }); // Wipe archive + close out beta. const archive = betaArchiveOf(betaId); if (await exists(archive)) await rm(archive, { recursive: true, force: true }); // Per-version test-failed transition. Skip a version if (a) another beta // still gates it at ready-promote/promoted, or (b) it's the currently- // promoted version of its script (already-live code isn't "failed" by // rejecting a redundant beta). for (const { script, versionId } of bundle) { const stillOk = db() .prepare( `SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id WHERE bs.script_version_id = ? AND b.id <> ? AND b.state IN ('ready-promote','promoted') LIMIT 1` ) .get(versionId, betaId); const isCurrent = script.current_version_id === versionId; if (!stillOk && !isCurrent) { db().prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`).run(versionId); } } emit({ type: 'beta-state', betaId, state: 'rolled-back' }); return { betaId, deploymentId, restored: [...restored].sort(), dropped: [...dropped].sort(), warnings }; } // ---------- promote ---------- export interface PromoteResult { betaId: number; deploymentId: string; landed: string[]; // unique servers that received at least one file scriptIds: number[]; awaitingActivation: number; failedServers: string[]; } /** Snapshot exactly the scope expansion and composed configuration reviewed by the operator. */ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configContent: Map } { const beta = getBeta(betaId); if (!beta) throw new BetaError('This beta no longer exists.', 'no-such-beta'); const bundle = listBetaScripts(betaId).map(row => { const script = getScript(row.script_id), version = getVersion(row.script_version_id); if (!script || !version) throw new BetaError('A selected file version is no longer available.', 'no-such-version'); return { script, version }; }); if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts'); const participants = listBetaServers(betaId); const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server)); const targets = (script: ScriptRow) => [...new Set([...tested, ...(script.scope_type && script.scope_value ? serversMatchingScope(script.scope_type, script.scope_value) : [])])].sort(); const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind, path: script.target_rel_path, versionId: version.id, version: version.version, release_version: version.release_version, sha256: version.sha256, scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) })); const names = [...new Set(files.flatMap(file => file.servers))].sort(); const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server)) .map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`); const servers = names.map(name => { const server = getServer(name); if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`); return { name, tested: tested.has(name), game: server?.game ?? null, map: server?.map ?? null }; }); const collapsing = bundle.filter(({ script }) => isCollapsingKind(script.kind)); const configContent = new Map(); const configWrites = [...new Set(collapsing.flatMap(({ script }) => targets(script)))].sort().map(server => { const row = getServer(server); const composed = row && composeCfgForServer(server, row.game, row.map, { inBetaBaseline: collapsing.find(({ script }) => script.kind === 'baseline')?.version, inBetaOverlays: collapsing.filter(({ script }) => script.kind === 'config') }); const error = composed ? null : `${server}: no baseline is available to compose its configuration.`; if (error) blockers.push(error); const content = composed ? Buffer.from(composed.text, 'utf8') : null; if (content) configContent.set(server, content); return { server, path: configFileOf(server), sha256: content ? sha256Buffer(content) : null, error }; }); const impact = { betaId, files, servers, configWrites, blockers, restartPolicy: 'additional-when-empty' as const }; return { preview: { ...impact, token: sha256Buffer(Buffer.from(JSON.stringify(impact))), createdAt: nowIso() }, configContent }; } export function previewBetaPromotion(betaId: number): BetaPromotionPreview { return promotionPlan(betaId).preview; } function promoteDeploymentId(betaId: number): string { const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); return `dep-${stamp}-promote-${betaId}`; } export async function promoteBeta(betaId: number, reviewedToken?: string): Promise { if (storedPlan(betaId)) { const { plannedBetaTick } = await import('../pipeline/beta-plan-runner.js'); await plannedBetaTick(); } return exclusive(betaId, () => promoteBetaInternal(betaId, reviewedToken)); } async function promoteBetaInternal(betaId: number, reviewedToken?: string): Promise { assertNativeActivationSettled(betaId); const test = storedPlan(betaId); if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote'); const beta = getBeta(betaId); if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta'); if (beta.state !== 'ready-promote') { throw new BetaError(`beta must be ready-promote (is ${beta.state})`, 'not-ready-promote'); } const reviewed = promotionPlan(betaId); const checkReview = () => { const current = previewBetaPromotion(betaId); if (current.token !== (reviewedToken ?? reviewed.preview.token)) { throw new BetaError('Promotion targets or versions changed. Review the updated plan before promoting.', 'preview-changed', current); } if (current.blockers.length) throw new BetaError(current.blockers.join(' '), 'invalid-state'); }; checkReview(); const bundleRows = listBetaScripts(betaId); if (bundleRows.length === 0) throw new BetaError('beta has no scripts', 'no-scripts'); const bundle = bundleRows.map((br) => ({ script: getScript(br.script_id)!, version: getVersion(br.script_version_id)! })); const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server)); // Split bundle into non-collapsing (per-file) and collapsing (composed cfg). const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind)); const collapsing = bundle.filter((b) => isCollapsingKind(b.script.kind)); // Build a per-(script, server) plan for non-collapsing artifacts. interface Plan { script: ScriptRow; version: ScriptVersionRow; server: string; priorVersionId: number | null; priorSha: string | null; priorContent: Buffer | null; hadFile: boolean; } const plans: Plan[] = []; const getSnapshotForPromote = db().prepare( `SELECT prior_content, prior_sha256 FROM beta_file_snapshots WHERE beta_id = ? AND server = ? AND rel_path = ?` ); for (const { script, version } of nonCollapsing) { const targets = reviewed.preview.files.find(file => file.id === script.id)!.servers; for (const name of targets) { const dest = destOfKind(name, script.kind); let plan: Plan; if (betaServerSet.has(name)) { const bss = getBetaServerScript(betaId, name, script.id); if (bss?.had_original) { const snap = getSnapshotForPromote.get(betaId, name, script.target_rel_path) as | { prior_content: Buffer; prior_sha256: string } | undefined; let priorContent: Buffer | null = null; let priorSha: string | null = null; if (snap) { priorContent = snap.prior_content; priorSha = snap.prior_sha256; } else if (bss.prior_version_id) { const pv = getVersion(bss.prior_version_id); if (pv) { priorContent = pv.content; priorSha = pv.sha256; } } plan = { script, version, server: name, priorVersionId: bss.prior_version_id, priorSha, priorContent, hadFile: true }; } else { plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false }; } } else { // Non-beta matching-scope server: current on-disk file (if any) // is the "before" state for a promote-then-rollback round trip. const targetPath = join(dest, script.target_rel_path); if (await exists(targetPath)) { const priorContent = await readFile(targetPath); const inv = getServerScript(name, script.target_rel_path); plan = { script, version, server: name, priorVersionId: inv?.script_version_id ?? null, priorSha: sha256Buffer(priorContent), priorContent, hadFile: true }; } else { plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false }; } } plans.push(plan); } } // Build durable activation work; live inventory changes only at activation. const serverErr = new Map(); // Collapsing config layer: one composed cfg per server. Union of beta // servers + scope-matching servers for every collapsing artifact. interface CfgPlan { server: string; priorContent: Buffer | null; priorSha: string | null; composedContent: Buffer; composedSha: string; hadFile: boolean; requiresActivation: boolean; } const cfgPlans: CfgPlan[] = []; if (collapsing.length > 0) { const cfgTargets = reviewed.preview.configWrites.map(write => write.server); for (const name of cfgTargets) { const srv = getServer(name); if (!srv) { serverErr.set(name, 'unknown server during config promote'); continue; } const composedBuf = reviewed.configContent.get(name); if (!composedBuf) { serverErr.set(name, `no baseline available for ${name} (game=${srv.game ?? '?'})`); continue; } const composedSha = sha256Buffer(composedBuf); const dest = destOfKind(name, 'config'); const targetRel = configFileOf(name); const targetPath = join(dest, targetRel); let priorContent: Buffer | null = null; let priorSha: string | null = null; let hadFile = false; // Beta server: preserve its original bytes captured at activation. if (betaServerSet.has(name)) { const snap = getSnapshotForPromote.get(betaId, name, targetRel) as | { prior_content: Buffer; prior_sha256: string } | undefined; if (snap) { priorContent = snap.prior_content; priorSha = snap.prior_sha256; hadFile = true; } } if (!betaServerSet.has(name) && await exists(targetPath)) { // Capture an initial preview; activation records the actual before image. priorContent = await readFile(targetPath); priorSha = sha256Buffer(priorContent); hadFile = true; } const requiresActivation = !betaServerSet.has(name) || !await exists(targetPath) || sha256Buffer(await readFile(targetPath)) !== composedSha; cfgPlans.push({ server: name, priorContent, priorSha, composedContent: composedBuf, composedSha, hadFile, requiresActivation }); } } // Recheck after asynchronous snapshot reads, before any mutation. Execution uses // the captured target lists/content so later membership changes never expand it. checkReview(); // A scope expansion from another promotion may have been queued while this // beta was active. Reassert the tested bytes after that older campaign, even // when they already match the live file, so it cannot overwrite this release. const queuedByServer = new Map(reviewed.preview.servers.map(server => [server.name, unresolvedCampaignWrites(server.name)])); const plansToActivate = new Set(plans.filter(p => !betaServerSet.has(p.server) || queuedByServer.get(p.server)?.some(row => row.kind === p.script.kind && row.rel_path === p.script.target_rel_path))); for (const cp of cfgPlans) { if (queuedByServer.get(cp.server)?.some(row => row.kind === 'config' && row.rel_path === configFileOf(cp.server))) cp.requiresActivation = true; } const activationServers = new Set([...Array.from(plansToActivate, plan => plan.server), ...cfgPlans.filter(plan => plan.requiresActivation).map(plan => plan.server)]); // Build the aggregate deployment. const depId = promoteDeploymentId(betaId); const startedAt = nowIso(); const insertDep = db().prepare( `INSERT INTO deployments (id, action, started_at, triggered_by, script_id, script_version_id, prior_version_id, beta_id) VALUES (?, 'promote', ?, 'ui', NULL, NULL, NULL, ?)` ); const insertDepScript = db().prepare( `INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id, kind) VALUES (?, ?, ?, ?, ?)` ); const insertDepServer = db().prepare( `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error) VALUES (?, ?, ?, ?, NULL, NULL, ?, 0, ?)` ); const insertDepFile = db().prepare( `INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ); const uniqueServers = Array.from(new Set([ ...plans.map((p) => p.server), ...cfgPlans.map((c) => c.server) ])); db().transaction(() => { insertDep.run(depId, startedAt, betaId); for (const { script, version } of bundle) { insertDepScript.run(depId, script.id, version.id, script.current_version_id, script.kind); } // Record the reviewed change. Scope expansion gets its actual before // image when activated; tested servers keep their pre-beta history. for (const p of plans) { const verb = p.hadFile ? 'updated' : 'added'; const file = insertDepFile.run( depId, p.server, verb, p.script.target_rel_path, p.priorSha, p.version.sha256, p.priorContent, p.version.content ); if (plansToActivate.has(p)) { queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: p.server, kind: p.script.kind, relPath: p.script.target_rel_path, content: p.version.content, scriptVersionId: p.version.id, inventorySource: `promote:${depId}`, deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(p.server) }); } } for (const cp of cfgPlans) { const verb = cp.hadFile ? 'updated' : 'added'; const file = insertDepFile.run( depId, cp.server, verb, configFileOf(cp.server), cp.priorSha, cp.composedSha, cp.priorContent, cp.composedContent ); if (cp.requiresActivation) { queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: cp.server, kind: 'config', relPath: configFileOf(cp.server), content: cp.composedContent, scriptVersionId: null, inventorySource: `promote:${depId}`, deploymentFileId: Number(file.lastInsertRowid), preserveBeforeImage: betaServerSet.has(cp.server) }); } } // deployment_servers — one row per unique server. Beta servers are // already restarted; scope-only servers still need the sweep to drain // and restart them. for (const name of uniqueServers) { const err = serverErr.get(name); if (err) { insertDepServer.run(depId, name, 0, 'failed', null, err); continue; } const filesChanged = plans.filter((p) => p.server === name).length + cfgPlans.filter((c) => c.server === name).length; if (!activationServers.has(name)) { const bsRestartedAt = db() .prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`) .get(betaId, name) as { restarted_at: string | null } | undefined; insertDepServer.run(depId, name, filesChanged, 'done', bsRestartedAt?.restarted_at ?? startedAt, null); } else { insertDepServer.run(depId, name, filesChanged, 'pending', null, null); } } // Update each script's current_version_id in the same transaction. for (const { script, version } of bundle) { db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); } })(); db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId); // The deployment history owns the pre-beta bytes for future rollback. db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId); const archive = betaArchiveOf(betaId); if (await exists(archive)) await rm(archive, { recursive: true, force: true }); const landed = Array.from(new Set([ ...plans.map((p) => p.server), ...cfgPlans.map((c) => c.server) ])).sort(); emit({ type: 'beta-state', betaId, state: 'promoted' }); emit({ type: 'deployment-opened', deploymentId: depId, action: 'promote', servers: landed }); return { betaId, deploymentId: depId, landed, scriptIds: bundle.map((b) => b.script.id), awaitingActivation: [...activationServers].filter(name => !serverErr.has(name)).length, failedServers: [...serverErr.keys()] }; } // ---------- helpers used by API layer ---------- // Latest of the script's versions — useful for the "open beta with this // version" default in the UI. export function latestVersionForScript(script: ScriptRow): ScriptVersionRow | null { const rows = listVersions(script.id); return rows[0] ?? null; } /** * Un-set a script's scope. Files are left on disk so drift picks up the * change on the next /deployments rescan and the operator reconciles from * there like any other drift — the same visible flow as manual drift. * * For scripts/plugins, we detach the inventory rows for this artifact's * versions so drift Pass 1 flags the on-disk copies as untracked. For * configs/baselines, no detach is needed: drift Pass 2 recomposes the * expected dedicated_sp.cfg against the current promoted overlays, and * scope=null naturally drops the overlay from the compose, surfacing the * disparity. Returns the servers whose inventory was detached (empty for * configs/baselines). */ export function unsetScopeAndDetach(scriptId: number): string[] { const script = getScript(scriptId); if (!script) throw new BetaError(`no such script (id=${scriptId})`, 'no-such-script'); setScope({ scriptId, scopeType: null, scopeValue: null }); const detached: string[] = []; if (script.kind === 'script' || script.kind === 'plugin') { const rows = db() .prepare( `SELECT server FROM server_scripts WHERE rel_path = ? AND script_version_id IN (SELECT id FROM script_versions WHERE script_id = ?)` ) .all(script.target_rel_path, scriptId) as Array<{ server: string }>; for (const { server } of rows) { removeScript(server, script.target_rel_path); detached.push(server); } } return detached; }