josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

multi script beta

Browse Source
This commit is contained in:
josh
2026-09-12 18:23:56 -04:00
parent 30e59c900c
commit 85bcb05a7d
16 changed files with 1053 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta.ts
+395 -276
View File
@@ -1,7 +1,7 @@
import { copyFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join, posix } from 'node:path';
import { betaArchiveOf, config, destOf } from '../config.js';
import { betaArchiveOf, destOf } from '../config.js';
import { db } from '../db/index.js';
import { betaFile, oldFile, sha256Buffer } from './files.js';
import { ensureDest, getServer, heldByBeta } from './servers.js';
@@ -17,13 +17,14 @@ import { recordScript, removeScript, getServerScript } from './inventory.js';
import { emit } from '../pipeline/bus.js';
import type { BetaStage } from '../pipeline/bus.js';
export type BetaTriggerKind = 'mark-required' | 'version-update' | 'unscoped';
export interface BetaRow {
id: number;
// Deprecated: the columns are still populated (NOT NULL on legacy schemas)
// from the first bundled script's identity so existing data continues to
// roundtrip, but new code reads beta_scripts for the authoritative list.
script_id: number;
script_version_id: number;
trigger_kind: BetaTriggerKind;
trigger_kind: string;
wait_for_health: number;
wait_for_match: number;
state: 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
@@ -35,8 +36,8 @@ export interface BetaRow {
export interface BetaServerRow {
beta_id: number;
server: string;
had_original: number;
prior_version_id: number | null;
had_original: number; // deprecated; per-script data lives in beta_server_scripts
prior_version_id: number | null; // deprecated
stage: BetaStage;
error: string | null;
deployed_at: string | null;
@@ -46,6 +47,20 @@ export interface BetaServerRow {
updated_at: string;
}
export interface BetaScriptRow {
beta_id: number;
script_id: number;
script_version_id: number;
}
export interface BetaServerScriptRow {
beta_id: number;
server: string;
script_id: number;
had_original: number;
prior_version_id: number | null;
}
export class BetaError extends Error {
constructor(
message: string,
@@ -53,6 +68,7 @@ export class BetaError extends Error {
| 'no-such-script'
| 'no-such-version'
| 'version-script-mismatch'
| 'no-scripts'
| 'no-servers'
| 'unknown-server'
| 'out-of-scope'
@@ -62,7 +78,9 @@ export class BetaError extends Error {
| 'not-ready-promote'
| 'not-running'
| 'dest-fail'
| 'invalid-state',
| 'invalid-state'
| 'duplicate-script'
| 'duplicate-target',
public readonly details?: unknown
) {
super(message);
@@ -92,79 +110,133 @@ export function listBetaServers(betaId: number): BetaServerRow[] {
.all(betaId) as BetaServerRow[];
}
export function listBetaScripts(betaId: number): BetaScriptRow[] {
return db()
.prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id')
.all(betaId) as BetaScriptRow[];
}
export function getBetaServerScript(betaId: number, server: string, scriptId: number): BetaServerScriptRow | undefined {
return db()
.prepare('SELECT * FROM beta_server_scripts WHERE beta_id = ? AND server = ? AND script_id = ?')
.get(betaId, server, scriptId) as BetaServerScriptRow | undefined;
}
// A script is "in a running beta" if beta_scripts joins to a running beta row.
// Existing callers only care about the running-beta id, so return that shape.
export function findRunningBetaForScript(scriptId: number): BetaRow | undefined {
return db()
.prepare(`SELECT * FROM betas WHERE script_id = ? AND state = 'running' LIMIT 1`)
.prepare(
`SELECT b.* FROM betas b
JOIN beta_scripts bs ON bs.beta_id = b.id
WHERE bs.script_id = ? AND b.state = 'running'
LIMIT 1`
)
.get(scriptId) as BetaRow | undefined;
}
// ---------- openBeta ----------
export interface OpenBetaInput {
export interface OpenBetaScript {
scriptId: number;
scriptVersionId: number;
}
export interface OpenBetaInput {
scripts: OpenBetaScript[];
servers: string[];
waitForHealth: boolean;
waitForMatch: boolean;
triggerKind: BetaTriggerKind;
notes?: string;
}
export interface OpenBetaResult {
betaId: number;
servers: string[];
scripts: OpenBetaScript[];
}
export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
const script = getScript(input.scriptId);
if (!script) throw new BetaError(`no such script (id=${input.scriptId})`, 'no-such-script');
const version = getVersion(input.scriptVersionId);
if (!version) throw new BetaError(`no such version (id=${input.scriptVersionId})`, 'no-such-version');
if (version.script_id !== script.id) {
throw new BetaError('version does not belong to this script', 'version-script-mismatch');
if (!input.scripts || input.scripts.length === 0) {
throw new BetaError('at least one script is required', 'no-scripts');
}
if (!input.servers || input.servers.length === 0) {
throw new BetaError('at least one server is required', 'no-servers');
}
// Look up server rows + validate scope up front.
const rows: Array<{ name: string; game: string | null; map: string | null }> = [];
// Resolve every (script, version) pair and validate belonging.
interface Resolved { script: ScriptRow; version: ScriptVersionRow }
const resolved: Resolved[] = [];
const seenScriptIds = new Set<number>();
const seenTargets = new Map<string, number>();
for (const s of input.scripts) {
if (seenScriptIds.has(s.scriptId)) {
throw new BetaError(`script ${s.scriptId} listed twice in one beta`, 'duplicate-script');
}
seenScriptIds.add(s.scriptId);
const script = getScript(s.scriptId);
if (!script) throw new BetaError(`no such script (id=${s.scriptId})`, 'no-such-script');
const version = getVersion(s.scriptVersionId);
if (!version) throw new BetaError(`no such version (id=${s.scriptVersionId})`, 'no-such-version');
if (version.script_id !== script.id) {
throw new BetaError(
`version ${s.scriptVersionId} does not belong to script ${s.scriptId}`,
'version-script-mismatch'
);
}
// Two scripts writing to the same target rel_path in one beta is an
// internal race — reject up front.
if (seenTargets.has(script.target_rel_path)) {
throw new BetaError(
`scripts ${seenTargets.get(script.target_rel_path)} and ${script.id} share target ${script.target_rel_path}`,
'duplicate-target'
);
}
seenTargets.set(script.target_rel_path, script.id);
resolved.push({ script, version });
}
// Look up server rows once.
const serverRows: Array<{ name: string; game: string | null; map: string | null }> = [];
for (const name of input.servers) {
const row = getServer(name);
if (!row) throw new BetaError(`unknown server: ${name}`, 'unknown-server');
rows.push({ name, game: row.game, map: row.map });
}
if (script.scope_type && script.scope_value) {
const outOfScope = rows.filter(
(r) => (script.scope_type === 'game' ? r.game : r.map) !== script.scope_value
);
if (outOfScope.length > 0) {
throw new BetaError(
`servers not in scope ${script.scope_type}=${script.scope_value}`,
'out-of-scope',
outOfScope.map((r) => r.name)
);
}
serverRows.push({ name, game: row.game, map: row.map });
}
// Per-server conflict check: is another running beta already holding this
// target on this server?
const conflicts: Array<{ server: string; betaId: number }> = [];
// Per-script scope validation. Aggregate all violations into one error.
const outOfScope: Array<{ scriptId: number; scope: string; servers: string[] }> = [];
for (const { script } of resolved) {
if (!script.scope_type || !script.scope_value) continue;
const bad = serverRows.filter(
(r) => (script.scope_type === 'game' ? r.game : r.map) !== script.scope_value
);
if (bad.length > 0) {
outOfScope.push({
scriptId: script.id,
scope: `${script.scope_type}=${script.scope_value}`,
servers: bad.map((r) => r.name)
});
}
}
if (outOfScope.length > 0) {
throw new BetaError('one or more servers are out of scope', 'out-of-scope', outOfScope);
}
// Per-(server, script) conflict + stale-.OLD check.
const conflicts: Array<{ server: string; scriptId: number; heldByBetaId: number }> = [];
const blocked: Array<{ server: string; ofile: string }> = [];
for (const name of input.servers) {
const held = heldByBeta(name);
const ownerId = held.get(script.target_rel_path);
if (ownerId) conflicts.push({ server: name, betaId: ownerId });
// If the target exists on-disk and .OLD already exists, refuse — that
// .OLD is either from a crashed prior beta or manual editing.
const dest = destOf(name);
const targetPath = join(dest, script.target_rel_path);
const backupPath = join(dest, oldFile(script.target_rel_path));
if (existsSync(targetPath) && existsSync(backupPath)) {
blocked.push({ server: name, ofile: oldFile(script.target_rel_path) });
for (const { script } of resolved) {
const owner = held.get(script.target_rel_path);
if (owner) conflicts.push({ server: name, scriptId: script.id, heldByBetaId: owner });
const dest = destOf(name);
const targetPath = join(dest, script.target_rel_path);
const backupPath = join(dest, oldFile(script.target_rel_path));
if (existsSync(targetPath) && existsSync(backupPath)) {
blocked.push({ server: name, ofile: oldFile(script.target_rel_path) });
}
}
}
if (conflicts.length > 0) throw new BetaError('target already held by another beta', 'conflict', conflicts);
@@ -176,49 +248,63 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
if (!ok.ok) throw new BetaError(`dest not usable for ${name}: ${ok.reason}`, 'dest-fail');
}
// Insert rows in a transaction — disk work happens after commit so we can
// await file I/O without holding the DB lock.
// DB writes in one transaction.
const primary = resolved[0]; // populates deprecated betas.* NOT NULL cols
const insertBeta = db().prepare(
`INSERT INTO betas (script_id, script_version_id, trigger_kind, wait_for_health, wait_for_match, state, notes)
VALUES (?, ?, ?, ?, ?, 'running', ?)`
VALUES (?, ?, 'bundle', ?, ?, 'running', ?)`
);
const insertBetaScript = db().prepare(
`INSERT INTO beta_scripts (beta_id, script_id, script_version_id) VALUES (?, ?, ?)`
);
const insertBetaServer = db().prepare(
`INSERT INTO beta_servers (beta_id, server, had_original, prior_version_id, stage, updated_at)
VALUES (?, ?, ?, ?, 'deploying', ?)`
VALUES (?, ?, 0, NULL, 'deploying', ?)`
);
const insertBetaServerScript = db().prepare(
`INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original, prior_version_id)
VALUES (?, ?, ?, ?, ?)`
);
let betaId!: number;
db().transaction(() => {
const res = insertBeta.run(
script.id,
version.id,
input.triggerKind,
primary.script.id,
primary.version.id,
input.waitForHealth ? 1 : 0,
input.waitForMatch ? 1 : 0,
input.notes ?? null
);
betaId = Number(res.lastInsertRowid);
for (const r of resolved) {
insertBetaScript.run(betaId, r.script.id, r.version.id);
}
for (const name of input.servers) {
const prior = getServerScript(name, script.target_rel_path);
insertBetaServer.run(
betaId,
name,
prior ? 1 : 0,
prior?.script_version_id ?? null,
nowIso()
);
insertBetaServer.run(betaId, name, nowIso());
for (const r of resolved) {
const prior = getServerScript(name, r.script.target_rel_path);
insertBetaServerScript.run(
betaId,
name,
r.script.id,
prior ? 1 : 0,
prior?.script_version_id ?? null
);
}
}
})();
// On-disk deploy: for each server, backup target → .OLD (if any) and write
// -beta.gsc from the version's content.
const bfile = betaFile(script.target_rel_path);
const ofile = oldFile(script.target_rel_path);
// Archive: one -beta.gsc per script.
const archive = betaArchiveOf(betaId);
await mkdir(archive, { recursive: true });
await writeFile(join(archive, posix.basename(bfile)), version.content);
for (const r of resolved) {
const bfile = betaFile(r.script.target_rel_path);
await writeFile(join(archive, posix.basename(bfile)), r.version.content);
}
// On-disk deploy per (server, script). If any script write fails on a
// server, mark the whole server 'failed' — we can't restart a partially
// deployed server safely. Remaining scripts on that server are skipped.
const updateStage = db().prepare(
`UPDATE beta_servers
SET stage = ?, error = ?, deployed_at = ?, updated_at = ?
@@ -227,38 +313,52 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
for (const name of input.servers) {
const dest = destOf(name);
const targetPath = join(dest, script.target_rel_path);
const backupPath = join(dest, ofile);
const betaPath = join(dest, bfile);
try {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (existsSync(targetPath)) {
await rename(targetPath, backupPath);
// Move inventory row: target → .OLD (keep prior sha/version).
const prior = getServerScript(name, script.target_rel_path);
removeScript(name, script.target_rel_path);
if (prior) {
recordScript(name, ofile, prior.script_version_id, prior.sha256, prior.size, `beta:${betaId}`);
let firstErr: string | null = null;
for (const { script, version } of resolved) {
if (firstErr) break;
const bfile = betaFile(script.target_rel_path);
const ofile = oldFile(script.target_rel_path);
const targetPath = join(dest, script.target_rel_path);
const backupPath = join(dest, ofile);
const betaPath = join(dest, bfile);
try {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (existsSync(targetPath)) {
await rename(targetPath, backupPath);
const prior = getServerScript(name, script.target_rel_path);
removeScript(name, script.target_rel_path);
if (prior) {
recordScript(name, ofile, prior.script_version_id, prior.sha256, prior.size, `beta:${betaId}`);
}
}
await writeFile(betaPath, version.content);
recordScript(name, bfile, version.id, version.sha256, version.size, `beta:${betaId}`);
} catch (err) {
firstErr = `${script.name}: ${(err as Error).message}`;
}
await writeFile(betaPath, version.content);
recordScript(name, bfile, version.id, version.sha256, version.size, `beta:${betaId}`);
const t = nowIso();
}
const t = nowIso();
if (firstErr) {
updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` });
} else {
updateStage.run('awaiting-empty', null, t, t, betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'awaiting-empty', error: null });
} catch (err) {
const t = nowIso();
const msg = (err as Error).message;
updateStage.run('failed', `deploy: ${msg}`, t, t, betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${msg}` });
}
}
emit({ type: 'beta-opened', betaId, scriptId: script.id, servers: input.servers });
emit({
type: 'beta-opened',
betaId,
scriptIds: resolved.map((r) => r.script.id),
servers: input.servers
});
return { betaId, servers: input.servers };
return {
betaId,
servers: input.servers,
scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id }))
};
}
// ---------- rollback ----------
@@ -276,51 +376,59 @@ export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
if (beta.state !== 'running' && beta.state !== 'ready-promote') {
throw new BetaError(`beta is ${beta.state}, cannot rollback`, 'invalid-state');
}
const script = getScript(beta.script_id)!;
const bfile = betaFile(script.target_rel_path);
const ofile = oldFile(script.target_rel_path);
const bundleRows = listBetaScripts(betaId);
if (bundleRows.length === 0) {
throw new BetaError('beta has no scripts', 'no-scripts');
}
const bundle = bundleRows.map((br) => ({
script: getScript(br.script_id)!,
versionId: br.script_version_id
}));
const rows = listBetaServers(betaId);
const restored: string[] = [];
const dropped: string[] = [];
const servers = listBetaServers(betaId);
const restored = new Set<string>();
const dropped = new Set<string>();
const warnings: string[] = [];
for (const bs of rows) {
for (const bs of servers) {
const dest = destOf(bs.server);
const betaPath = join(dest, bfile);
const backupPath = join(dest, ofile);
const targetPath = join(dest, script.target_rel_path);
for (const { script } of bundle) {
const bfile = betaFile(script.target_rel_path);
const ofile = oldFile(script.target_rel_path);
const betaPath = join(dest, bfile);
const backupPath = join(dest, ofile);
const targetPath = join(dest, script.target_rel_path);
// Remove -beta.gsc regardless of stage.
if (existsSync(betaPath)) await rm(betaPath, { force: true });
removeScript(bs.server, bfile);
// Remove -beta.gsc regardless of stage.
if (existsSync(betaPath)) await rm(betaPath, { force: true });
removeScript(bs.server, bfile);
if (bs.had_original) {
if (existsSync(backupPath)) {
await rename(backupPath, targetPath);
removeScript(bs.server, ofile);
// Restore inventory row for target with prior version bytes.
const prior = bs.prior_version_id ? getVersion(bs.prior_version_id) : null;
if (prior) {
recordScript(bs.server, script.target_rel_path, prior.id, prior.sha256, prior.size, `rollback:${betaId}`);
const bss = getBetaServerScript(betaId, bs.server, script.id);
if (bss?.had_original) {
if (existsSync(backupPath)) {
await rename(backupPath, targetPath);
removeScript(bs.server, ofile);
const prior = bss.prior_version_id ? getVersion(bss.prior_version_id) : null;
if (prior) {
recordScript(bs.server, script.target_rel_path, prior.id, prior.sha256, prior.size, `rollback:${betaId}`);
} else {
const buf = await readFile(targetPath);
recordScript(
bs.server,
script.target_rel_path,
null,
sha256Buffer(buf),
buf.byteLength,
`rollback:${betaId}`
);
}
restored.add(bs.server);
} else {
// We had_original but lost the prior version reference — hash the file to record what's there.
const buf = await readFile(targetPath);
recordScript(
bs.server,
script.target_rel_path,
null,
sha256Buffer(buf),
buf.byteLength,
`rollback:${betaId}`
);
warnings.push(`${bs.server}/${script.name}: no ${ofile} to restore`);
}
restored.push(bs.server);
} else {
warnings.push(`${bs.server}: no ${ofile} to restore`);
dropped.add(bs.server);
}
} else {
dropped.push(bs.server);
}
}
@@ -332,37 +440,44 @@ export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
.prepare(`UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?`)
.run(nowIso(), betaId);
// Mark the version test-failed unless another beta for the same version
// still gates it as tested/promoted. Skip if the version is the currently-
// promoted one — that's authoritative, not a failure signal.
const stillOk = db()
.prepare(
`SELECT 1 FROM betas
WHERE script_version_id = ?
AND id <> ?
AND state IN ('ready-promote','promoted')
LIMIT 1`
)
.get(beta.script_version_id, betaId);
const isCurrent = script.current_version_id === beta.script_version_id;
if (!stillOk && !isCurrent) {
db()
.prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`)
.run(beta.script_version_id);
// Per-version test-failed transition. Skip a version if (a) another beta
// still gates it at ready-promote/promoted, or (b) it's the currently-
// promoted version of its script (already-live code isn't "failed" by
// rejecting a redundant beta).
for (const { script, versionId } of bundle) {
const stillOk = db()
.prepare(
`SELECT 1 FROM beta_scripts bs
JOIN betas b ON b.id = bs.beta_id
WHERE bs.script_version_id = ?
AND b.id <> ?
AND b.state IN ('ready-promote','promoted')
LIMIT 1`
)
.get(versionId, betaId);
const isCurrent = script.current_version_id === versionId;
if (!stillOk && !isCurrent) {
db().prepare(`UPDATE script_versions SET state = 'test-failed' WHERE id = ?`).run(versionId);
}
}
emit({ type: 'beta-state', betaId, state: 'rolled-back' });
return { betaId, restored, dropped, warnings };
return {
betaId,
restored: [...restored].sort(),
dropped: [...dropped].sort(),
warnings
};
}
// ---------- promote ----------
export interface PromoteResult {
betaId: number;
landed: string[]; // servers we wrote the promoted target to
versionId: number;
deploymentId: string;
landed: string[]; // unique servers that received at least one file
scriptIds: number[];
}
function promoteDeploymentId(betaId: number): string {
@@ -376,127 +491,114 @@ export async function promoteBeta(betaId: number): Promise<PromoteResult> {
if (beta.state !== 'ready-promote') {
throw new BetaError(`beta must be ready-promote (is ${beta.state})`, 'not-ready-promote');
}
const script = getScript(beta.script_id)!;
const version = getVersion(beta.script_version_id)!;
const bfile = betaFile(script.target_rel_path);
const ofile = oldFile(script.target_rel_path);
const bundleRows = listBetaScripts(betaId);
if (bundleRows.length === 0) throw new BetaError('beta has no scripts', 'no-scripts');
const bundle = bundleRows.map((br) => ({
script: getScript(br.script_id)!,
version: getVersion(br.script_version_id)!
}));
const betaServerSet = new Set(listBetaServers(betaId).map((r) => r.server));
// For scoped scripts, promotion applies to every matching server (whether
// or not they were in the beta). For unscoped, just the beta servers.
const scopeServers = script.scope_type && script.scope_value
? serversMatchingScope(script.scope_type, script.scope_value)
: [];
const allTargets = Array.from(new Set([...betaServerSet, ...scopeServers]));
// Snapshot what's on-disk at every target BEFORE we write, so the promote
// deployment can be rolled back later. For beta servers the real "before"
// state is the .OLD file (or nothing), not the current -beta.gsc.
interface Snapshot {
// Build a per-(script, server) plan: prior snapshot + intended write.
interface Plan {
script: ScriptRow;
version: ScriptVersionRow;
server: string;
priorVersionId: number | null;
priorSha: string | null;
priorSize: number | null;
priorContent: Buffer | null;
hadFile: boolean;
}
const snapshots: Snapshot[] = [];
for (const name of allTargets) {
const dest = destOf(name);
if (betaServerSet.has(name)) {
// Prior state is captured in the beta_servers row + .OLD file.
const bs = db()
.prepare(`SELECT had_original, prior_version_id FROM beta_servers WHERE beta_id = ? AND server = ?`)
.get(betaId, name) as { had_original: number; prior_version_id: number | null } | undefined;
if (bs?.had_original) {
const backupPath = join(dest, ofile);
let priorContent: Buffer | null = null;
let priorSha: string | null = null;
let priorSize: number | null = null;
if (existsSync(backupPath)) {
priorContent = await readFile(backupPath);
priorSha = sha256Buffer(priorContent);
priorSize = priorContent.byteLength;
} else if (bs.prior_version_id) {
const pv = getVersion(bs.prior_version_id);
if (pv) {
priorContent = pv.content;
priorSha = pv.sha256;
priorSize = pv.size;
const plans: Plan[] = [];
for (const { script, version } of bundle) {
const scopeServers = script.scope_type && script.scope_value
? serversMatchingScope(script.scope_type, script.scope_value)
: [];
const targets = new Set<string>([...betaServerSet, ...scopeServers]);
for (const name of targets) {
let plan: Plan;
if (betaServerSet.has(name)) {
const bss = getBetaServerScript(betaId, name, script.id);
if (bss?.had_original) {
const dest = destOf(name);
const backupPath = join(dest, oldFile(script.target_rel_path));
let priorContent: Buffer | null = null;
let priorSha: string | null = null;
if (existsSync(backupPath)) {
priorContent = await readFile(backupPath);
priorSha = sha256Buffer(priorContent);
} else if (bss.prior_version_id) {
const pv = getVersion(bss.prior_version_id);
if (pv) { priorContent = pv.content; priorSha = pv.sha256; }
}
plan = {
script, version, server: name,
priorVersionId: bss.prior_version_id, priorSha, priorContent, hadFile: true
};
} else {
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
snapshots.push({
server: name,
priorVersionId: bs.prior_version_id,
priorSha,
priorSize,
priorContent,
hadFile: true
});
} else {
snapshots.push({ server: name, priorVersionId: null, priorSha: null, priorSize: null, priorContent: null, hadFile: false });
}
} else {
// Non-beta matching-scope server — the current on-disk target is
// its "before".
const targetPath = join(dest, script.target_rel_path);
if (existsSync(targetPath)) {
const priorContent = await readFile(targetPath);
const inv = getServerScript(name, script.target_rel_path);
snapshots.push({
server: name,
priorVersionId: inv?.script_version_id ?? null,
priorSha: sha256Buffer(priorContent),
priorSize: priorContent.byteLength,
priorContent,
hadFile: true
});
} else {
snapshots.push({ server: name, priorVersionId: null, priorSha: null, priorSize: null, priorContent: null, hadFile: false });
// Non-beta matching-scope server: current on-disk file (if any)
// is the "before" state for a promote-then-rollback round trip.
const dest = destOf(name);
const targetPath = join(dest, script.target_rel_path);
if (existsSync(targetPath)) {
const priorContent = await readFile(targetPath);
const inv = getServerScript(name, script.target_rel_path);
plan = {
script, version, server: name,
priorVersionId: inv?.script_version_id ?? null,
priorSha: sha256Buffer(priorContent),
priorContent, hadFile: true
};
} else {
plan = { script, version, server: name, priorVersionId: null, priorSha: null, priorContent: null, hadFile: false };
}
}
plans.push(plan);
}
}
// Now write files. Failures are per-server and don't abort the promote —
// the deployment record captures who succeeded and who didn't.
const landed: string[] = [];
const failures = new Map<string, string>();
for (const name of allTargets) {
const dest = destOf(name);
const targetPath = join(dest, script.target_rel_path);
// Write files. Track per-plan success and per-server first-error.
const planWritten = new Map<Plan, boolean>();
const serverErr = new Map<string, string>();
for (const p of plans) {
const dest = destOf(p.server);
const bfile = betaFile(p.script.target_rel_path);
const ofile = oldFile(p.script.target_rel_path);
const targetPath = join(dest, p.script.target_rel_path);
const betaPath = join(dest, bfile);
const backupPath = join(dest, ofile);
try {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (existsSync(betaPath)) {
await rm(betaPath, { force: true });
removeScript(name, bfile);
}
if (existsSync(backupPath)) {
await rm(backupPath, { force: true });
removeScript(name, ofile);
}
await writeFile(targetPath, version.content);
landed.push(name);
await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true });
if (existsSync(betaPath)) { await rm(betaPath, { force: true }); removeScript(p.server, bfile); }
if (existsSync(backupPath)) { await rm(backupPath, { force: true }); removeScript(p.server, ofile); }
await writeFile(targetPath, p.version.content);
planWritten.set(p, true);
} catch (err) {
const msg = (err as Error).message;
failures.set(name, msg);
console.error(`[promote] ${name} failed:`, err);
planWritten.set(p, false);
const msg = `${p.script.name}: ${(err as Error).message}`;
if (!serverErr.has(p.server)) serverErr.set(p.server, msg);
console.error(`[promote] ${p.server}/${p.script.name} failed:`, err);
}
}
// Build the deployment record. Only landed servers get deployment_files
// rows — a failed write leaves the prior file on disk, so recording it as
// if it changed would corrupt any future rollback.
// Build the aggregate deployment.
const depId = promoteDeploymentId(betaId);
const startedAt = nowIso();
const priorScriptVersionId = script.current_version_id;
const insertDep = db().prepare(
`INSERT INTO deployments
(id, action, started_at, triggered_by, script_id, script_version_id, prior_version_id, beta_id)
VALUES (?, 'promote', ?, 'ui', ?, ?, ?, ?)`
VALUES (?, 'promote', ?, 'ui', NULL, NULL, NULL, ?)`
);
const insertDepScript = db().prepare(
`INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id)
VALUES (?, ?, ?, ?)`
);
const insertDepServer = db().prepare(
`INSERT INTO deployment_servers
@@ -509,59 +611,77 @@ export async function promoteBeta(betaId: number): Promise<PromoteResult> {
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
);
const uniqueServers = Array.from(new Set(plans.map((p) => p.server)));
db().transaction(() => {
insertDep.run(depId, startedAt, script.id, version.id, priorScriptVersionId, betaId);
for (const snap of snapshots) {
if (failures.has(snap.server)) {
// Record the failure but no file row — nothing changed on disk.
insertDepServer.run(depId, snap.server, 0, 'failed', null, failures.get(snap.server)!);
continue;
}
const verb = snap.hadFile ? 'updated' : 'added';
insertDep.run(depId, startedAt, betaId);
for (const { script, version } of bundle) {
insertDepScript.run(depId, script.id, version.id, script.current_version_id);
}
// deployment_files — one row per successful (script, server) write.
for (const p of plans) {
if (!planWritten.get(p)) continue;
const verb = p.hadFile ? 'updated' : 'added';
insertDepFile.run(
depId,
snap.server,
p.server,
verb,
script.target_rel_path,
snap.priorSha,
version.sha256,
snap.priorContent,
version.content
p.script.target_rel_path,
p.priorSha,
p.version.sha256,
p.priorContent,
p.version.content
);
// Beta servers are already at the new bytes AND already restarted
// during the beta's own state machine. Non-beta matching servers
// got the file just now and still need to drain-and-restart, which
// the deployment sweep handles.
if (betaServerSet.has(snap.server)) {
recordScript(
p.server,
p.script.target_rel_path,
p.version.id,
p.version.sha256,
p.version.size,
`promote:${depId}`
);
}
// deployment_servers — one row per unique server. Beta servers are
// already restarted; scope-only servers still need the sweep to drain
// and restart them.
for (const name of uniqueServers) {
const err = serverErr.get(name);
if (err) {
insertDepServer.run(depId, name, 0, 'failed', null, err);
continue;
}
const filesChanged = plans.filter((p) => p.server === name && planWritten.get(p)).length;
if (betaServerSet.has(name)) {
const bsRestartedAt = db()
.prepare(`SELECT restarted_at FROM beta_servers WHERE beta_id = ? AND server = ?`)
.get(betaId, snap.server) as { restarted_at: string | null } | undefined;
insertDepServer.run(depId, snap.server, 1, 'done', bsRestartedAt?.restarted_at ?? startedAt, null);
.get(betaId, name) as { restarted_at: string | null } | undefined;
insertDepServer.run(depId, name, filesChanged, 'done', bsRestartedAt?.restarted_at ?? startedAt, null);
} else {
insertDepServer.run(depId, snap.server, 1, 'pending', null, null);
insertDepServer.run(depId, name, filesChanged, 'pending', null, null);
}
recordScript(snap.server, script.target_rel_path, version.id, version.sha256, version.size, `promote:${depId}`);
}
// Update each script's current_version_id in the same transaction.
for (const { script, version } of bundle) {
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
}
})();
// Update the script's canonical version and close out the beta + wipe archive.
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
db().prepare(`UPDATE betas SET state = 'promoted', closed_at = ? WHERE id = ?`).run(nowIso(), betaId);
const archive = betaArchiveOf(betaId);
if (existsSync(archive)) await rm(archive, { recursive: true, force: true });
emit({ type: 'beta-state', betaId, state: 'promoted' });
emit({
type: 'deployment-opened',
deploymentId: depId,
action: 'promote',
servers: landed
});
const landed = Array.from(new Set(plans.filter((p) => planWritten.get(p)).map((p) => p.server))).sort();
return { betaId, landed, versionId: version.id, deploymentId: depId };
emit({ type: 'beta-state', betaId, state: 'promoted' });
emit({ type: 'deployment-opened', deploymentId: depId, action: 'promote', servers: landed });
return {
betaId,
deploymentId: depId,
landed,
scriptIds: bundle.map((b) => b.script.id)
};
}
// ---------- helpers used by API layer ----------
@@ -575,8 +695,7 @@ export function latestVersionForScript(script: ScriptRow): ScriptVersionRow | nu
/**
* Un-set a script's scope. When scope is being removed and matching servers
* currently have the script, remove it from them for consistency (per plan:
* "un-marking auto-removes from matching servers, no beta"). Returns the
* currently have the script, remove it from them for consistency. Returns the
* server names that had files deleted.
*/
export async function unsetScopeAndCleanup(scriptId: number): Promise<string[]> {
src/lib/server/fleet/rollback.ts
+78 -39
View File
@@ -7,19 +7,11 @@ import { emit } from '../pipeline/bus.js';
import { recordScript, removeScript } from './inventory.js';
import { getVersion } from './scripts.js';
// Roll a promoted script back to whatever version was current before the
// promote landed. Uses the deployment_files snapshots already captured at
// promote time — no on-disk archaeology needed.
//
// Semantics:
// - Reverses only the file writes recorded in deployment_files. Servers that
// failed at promote time (no file row) are left alone.
// - Restores scripts.current_version_id to the promote row's prior_version_id.
// - Marks the original promote as rolled_back_by = <new deployment id>.
// - Records a fresh action='rollback' deployment with pending server rows so
// the sweep drains-and-restarts every affected server. Beta servers that
// were still running the promoted bytes get restarted the same as any
// other; the beta itself is not touched (it stays in state='promoted').
// Roll a promoted deployment back to whatever versions were current before
// the promote landed. Reads deployment_scripts as the source of truth for
// which scripts were involved (and their prior versions); falls back to the
// deprecated deployments.script_id/version_id/prior_version_id columns for
// pre-multi-script legacy rows.
export class RollbackError extends Error {
constructor(message: string, public readonly kind: 'no-such-deployment' | 'not-promote' | 'already-rolled-back') {
@@ -36,6 +28,13 @@ interface DeploymentRow {
rolled_back_by: string | null;
}
interface DepScriptRow {
deployment_id: string;
script_id: number;
script_version_id: number;
prior_version_id: number | null;
}
interface FileRow {
id: number;
server: string;
@@ -47,6 +46,11 @@ interface FileRow {
content_after: Buffer | null;
}
interface ScriptTargetRow {
id: number;
target_rel_path: string;
}
function nowIso(): string {
return new Date().toISOString();
}
@@ -57,10 +61,9 @@ function rollbackDeploymentId(promoteId: string): string {
}
export interface RollbackDeploymentResult {
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptId: number | null;
priorVersionId: number | null;
deploymentId: string; // the new rollback deployment
reverses: string; // the promote deployment id
scriptIds: number[]; // scripts whose current_version_id got reverted
affectedServers: string[];
}
@@ -70,8 +73,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
FROM deployments WHERE id = ?`)
.get(promoteId) as DeploymentRow | undefined;
if (!dep) throw new RollbackError(`no such deployment ${promoteId}`, 'no-such-deployment');
if (dep.action !== 'promote') throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote');
if (dep.rolled_back_by) throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back');
if (dep.action !== 'promote') {
throw new RollbackError(`deployment ${promoteId} is action='${dep.action}', not 'promote'`, 'not-promote');
}
if (dep.rolled_back_by) {
throw new RollbackError(`deployment ${promoteId} already rolled back by ${dep.rolled_back_by}`, 'already-rolled-back');
}
// Source of truth for which scripts (and prior versions) this promote
// touched. Prefer deployment_scripts; fall back to the deprecated
// singular columns for legacy single-script deployments.
let scripts = db()
.prepare(`SELECT * FROM deployment_scripts WHERE deployment_id = ? ORDER BY script_id`)
.all(promoteId) as DepScriptRow[];
if (scripts.length === 0 && dep.script_id !== null && dep.script_version_id !== null) {
scripts = [{
deployment_id: promoteId,
script_id: dep.script_id,
script_version_id: dep.script_version_id,
prior_version_id: dep.prior_version_id
}];
}
const fileRows = db()
.prepare(
@@ -80,10 +102,26 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
)
.all(promoteId) as FileRow[];
// Map rel_path → scriptId (and prior_version_id) so we can restore the
// inventory row's script_version_id correctly during file restore.
const scriptByTarget = new Map<string, DepScriptRow>();
if (scripts.length > 0) {
const placeholders = scripts.map(() => '?').join(',');
const targetRows = db()
.prepare(`SELECT id, target_rel_path FROM scripts WHERE id IN (${placeholders})`)
.all(...scripts.map((s) => s.script_id)) as ScriptTargetRow[];
const targetById = new Map<number, string>();
for (const t of targetRows) targetById.set(t.id, t.target_rel_path);
for (const s of scripts) {
const t = targetById.get(s.script_id);
if (t) scriptByTarget.set(t, s);
}
}
const newDepId = rollbackDeploymentId(promoteId);
const startedAt = nowIso();
// Group by server so each server gets exactly one deployment_servers row.
// Group files by server so each server gets exactly one deployment_servers row.
const perServer = new Map<string, FileRow[]>();
for (const f of fileRows) {
const arr = perServer.get(f.server);
@@ -100,14 +138,15 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
for (const f of files) {
const abs = join(dest, f.rel_path);
await mkdir(join(dest, dirname(f.rel_path)), { recursive: true });
const owningScript = scriptByTarget.get(f.rel_path);
if (f.verb === 'added') {
// Promote created this file — undo by removing it.
if (existsSync(abs)) await rm(abs, { force: true });
removeScript(server, f.rel_path);
} else if (f.verb === 'updated' && f.content_before) {
await writeFile(abs, f.content_before);
// Restore inventory to the prior version if we can identify it.
const priorVer = dep.prior_version_id ? getVersion(dep.prior_version_id) : null;
const priorVer = owningScript?.prior_version_id
? getVersion(owningScript.prior_version_id)
: null;
recordScript(
server,
f.rel_path,
@@ -117,7 +156,6 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
`rollback:${newDepId}`
);
} else if (f.verb === 'removed' && f.content_before) {
// Should not occur on a promote row, but handle symmetrically.
await writeFile(abs, f.content_before);
recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`);
}
@@ -132,15 +170,17 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const insertDep = db().prepare(
`INSERT INTO deployments
(id, action, started_at, triggered_by, script_id, script_version_id, prior_version_id, beta_id)
VALUES (?, 'rollback', ?, 'ui', ?, ?, ?, NULL)`
VALUES (?, 'rollback', ?, 'ui', NULL, NULL, NULL, NULL)`
);
const insertDepScript = db().prepare(
`INSERT INTO deployment_scripts (deployment_id, script_id, script_version_id, prior_version_id)
VALUES (?, ?, ?, ?)`
);
const insertDepServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
VALUES (?, ?, ?, ?, NULL, NULL, NULL, 0, ?)`
);
// Mirror the promote's deployment_files into the rollback row so it's
// browsable as a diff (before/after swapped).
const insertDepFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
@@ -148,13 +188,13 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
db().transaction(() => {
insertDep.run(
newDepId,
startedAt,
dep.script_id,
dep.prior_version_id, // this rollback lands the *prior* version, so that's now current
dep.script_version_id, // what was in place immediately before this rollback
);
insertDep.run(newDepId, startedAt);
// Mirror the promote's script list — swapped: the "current" version
// after rollback is the promote's prior; the "prior" of the rollback
// is what was current up until this call (i.e. the promote's version).
for (const s of scripts) {
insertDepScript.run(newDepId, s.script_id, s.prior_version_id ?? s.script_version_id, s.script_version_id);
}
for (const [server, files] of perServer) {
if (perServerFailures.has(server)) {
insertDepServer.run(newDepId, server, 0, 'failed', perServerFailures.get(server)!);
@@ -179,9 +219,9 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
}
}
db().prepare(`UPDATE deployments SET rolled_back_by = ? WHERE id = ?`).run(newDepId, promoteId);
// Restore the script's current version pointer.
if (dep.script_id !== null) {
db().prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`).run(dep.prior_version_id, dep.script_id);
// Restore each affected script's current_version_id.
for (const s of scripts) {
db().prepare(`UPDATE scripts SET current_version_id = ? WHERE id = ?`).run(s.prior_version_id, s.script_id);
}
})();
@@ -190,8 +230,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
return {
deploymentId: newDepId,
reverses: promoteId,
scriptId: dep.script_id,
priorVersionId: dep.prior_version_id,
scriptIds: scripts.map((s) => s.script_id),
affectedServers
};
}
src/lib/server/fleet/scripts.ts
+10 -2
View File
@@ -207,7 +207,11 @@ export function deleteVersion(versionId: number): void {
}
const beta = db()
.prepare(`SELECT id FROM betas WHERE script_version_id = ? AND state = 'running'`)
.prepare(
`SELECT b.id FROM betas b
JOIN beta_scripts bs ON bs.beta_id = b.id
WHERE bs.script_version_id = ? AND b.state = 'running' LIMIT 1`
)
.get(versionId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta');
@@ -221,7 +225,11 @@ export function deleteScript(scriptId: number): void {
throw new ScriptError('un-set scope before deleting a script', 'has-scope');
}
const beta = db()
.prepare(`SELECT id FROM betas WHERE script_id = ? AND state = 'running'`)
.prepare(
`SELECT b.id FROM betas b
JOIN beta_scripts bs ON bs.beta_id = b.id
WHERE bs.script_id = ? AND b.state = 'running' LIMIT 1`
)
.get(scriptId) as { id: number } | undefined;
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta');
db().prepare('DELETE FROM scripts WHERE id = ?').run(scriptId);
src/lib/server/fleet/servers.ts
+10 -8
View File
@@ -153,10 +153,11 @@ export function heldByBeta(server: string): Map<string, number> {
const rows = db()
.prepare(
`SELECT s.target_rel_path AS target, b.id AS beta_id
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
JOIN scripts s ON s.id = b.script_id
WHERE bs.server = ? AND b.state = 'running'`
FROM beta_servers bsv
JOIN betas b ON b.id = bsv.beta_id
JOIN beta_scripts bs ON bs.beta_id = b.id
JOIN scripts s ON s.id = bs.script_id
WHERE bsv.server = ? AND b.state = 'running'`
)
.all(server) as Array<{ target: string; beta_id: number }>;
const map = new Map<string, number>();
@@ -171,10 +172,11 @@ export function ownerOf(server: string, target: string): number | null {
const row = db()
.prepare(
`SELECT b.id AS beta_id
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
JOIN scripts s ON s.id = b.script_id
WHERE bs.server = ? AND s.target_rel_path = ? AND b.state = 'running'
FROM beta_servers bsv
JOIN betas b ON b.id = bsv.beta_id
JOIN beta_scripts bs ON bs.beta_id = b.id
JOIN scripts s ON s.id = bs.script_id
WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state = 'running'
LIMIT 1`
)
.get(server, target) as { beta_id: number } | undefined;