Add configurable beta test plans and staged rollout
This commit is contained in:
@@ -129,6 +129,54 @@ CREATE TABLE IF NOT EXISTS beta_servers (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_beta_servers_stage ON beta_servers(stage);
|
||||
|
||||
-- Immutable test configuration, durable attempts and append-only evidence.
|
||||
-- Keeping test outcome separate preserves artifact holds until explicit closure.
|
||||
CREATE TABLE IF NOT EXISTS beta_test_plans (
|
||||
beta_id INTEGER PRIMARY KEY REFERENCES betas(id) ON DELETE CASCADE,
|
||||
plan TEXT NOT NULL,
|
||||
outcome TEXT NOT NULL DEFAULT 'active',
|
||||
reason TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS beta_test_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
server TEXT NOT NULL REFERENCES servers(name),
|
||||
phase INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
observing_at TEXT,
|
||||
retired_at TEXT,
|
||||
metrics TEXT NOT NULL DEFAULT '{}',
|
||||
cursors TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS beta_test_coverage (
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
server TEXT NOT NULL,
|
||||
game TEXT,
|
||||
map TEXT,
|
||||
PRIMARY KEY (beta_id, server)
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_beta_test_current_run ON beta_test_runs(beta_id, server) WHERE retired_at IS NULL;
|
||||
CREATE TABLE IF NOT EXISTS beta_test_evidence (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id TEXT NOT NULL REFERENCES beta_test_runs(id) ON DELETE CASCADE,
|
||||
event_id TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
UNIQUE(run_id, event_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_beta_evidence_run ON beta_test_evidence(run_id, id);
|
||||
CREATE TABLE IF NOT EXISTS beta_file_changes (
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
server TEXT NOT NULL,
|
||||
rel_path TEXT NOT NULL,
|
||||
PRIMARY KEY (beta_id, server, rel_path)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS beta_test_presets (
|
||||
name TEXT PRIMARY KEY,
|
||||
plan TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Many-to-many: which scripts (and pinned versions) are bundled into a beta.
|
||||
-- Replaces the deprecated betas.script_id / betas.script_version_id columns.
|
||||
CREATE TABLE IF NOT EXISTS beta_scripts (
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { BetaPlan, RequirementProgress } from '../../beta-plan.js';
|
||||
|
||||
export interface SessionEvidence {
|
||||
started: number;
|
||||
sequence: number;
|
||||
ended: boolean;
|
||||
rounds: Record<string, number>;
|
||||
peakPlayers: number;
|
||||
}
|
||||
export interface TestMetrics {
|
||||
values: Record<string, number>;
|
||||
sessions: Record<string, SessionEvidence>;
|
||||
manual: Record<string, string>;
|
||||
bootId?: string;
|
||||
boots: string[];
|
||||
restarts: number;
|
||||
lastSample?: number;
|
||||
wasHealthy?: boolean;
|
||||
lastPlayers?: number;
|
||||
outages: number;
|
||||
logErrors: number;
|
||||
logAvailable?: boolean;
|
||||
streaks: Record<string, number>;
|
||||
}
|
||||
export const emptyMetrics = (): TestMetrics => ({ values: {}, sessions: {}, manual: {}, boots: [], restarts: 0, outages: 0, logErrors: 0, streaks: {} });
|
||||
|
||||
export interface TestSignal {
|
||||
event: 'boot' | 'session_start' | 'round_change' | 'match_end' | 'feature';
|
||||
bootId: string;
|
||||
sessionId: string;
|
||||
sequence: number;
|
||||
round: number;
|
||||
players: number;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/** Events are accepted once, in session order; a match must start in this run. */
|
||||
export function applySignal(plan: BetaPlan, metrics: TestMetrics, signal: TestSignal, now: number): boolean {
|
||||
if (signal.event === 'boot') {
|
||||
if (metrics.boots.includes(signal.bootId)) return false;
|
||||
if (metrics.bootId) metrics.restarts++;
|
||||
metrics.boots.push(signal.bootId);
|
||||
metrics.bootId = signal.bootId;
|
||||
return true;
|
||||
}
|
||||
if (metrics.bootId !== signal.bootId) return false;
|
||||
const key = `${signal.bootId}:${signal.sessionId}`;
|
||||
if (signal.event === 'session_start') {
|
||||
if (metrics.sessions[key]) return false;
|
||||
metrics.sessions[key] = { started: now, sequence: signal.sequence, ended: false, rounds: {}, peakPlayers: signal.players };
|
||||
return true;
|
||||
}
|
||||
const session = metrics.sessions[key];
|
||||
if (!session || session.ended || signal.sequence <= session.sequence) return false;
|
||||
session.sequence = signal.sequence;
|
||||
session.peakPlayers = Math.max(session.peakPlayers, signal.players);
|
||||
for (const r of plan.requirements) {
|
||||
const current = metrics.values[r.id] ?? 0;
|
||||
if (signal.event === 'round_change' && signal.players >= r.minPlayers) {
|
||||
session.rounds[r.id] = Math.max(session.rounds[r.id] ?? 0, signal.round);
|
||||
if (r.kind === 'round') metrics.values[r.id] = Math.max(current, signal.round);
|
||||
}
|
||||
if (signal.event === 'match_end' && r.kind === 'matches' && (r.minRound > 0 || r.minPlayers > 1)
|
||||
&& (session.rounds[r.id] ?? 0) >= r.minRound && session.peakPlayers >= r.minPlayers
|
||||
&& now - session.started >= r.minMinutes * 60_000) metrics.values[r.id] = current + 1;
|
||||
if (signal.event === 'feature' && r.kind === 'feature' && signal.name === r.event
|
||||
&& signal.players >= r.minPlayers && signal.round >= r.minRound) metrics.values[r.id] = current + 1;
|
||||
}
|
||||
if (signal.event === 'match_end') session.ended = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Sample both ends of an interval. Monitoring gaps never earn playtime. */
|
||||
export function applySample(plan: BetaPlan, m: TestMetrics, at: number, healthy: boolean, players: number): void {
|
||||
if (m.lastSample !== undefined && at <= m.lastSample) return;
|
||||
const gap = m.lastSample === undefined ? Infinity : at - m.lastSample;
|
||||
const contiguous = gap <= 60_000;
|
||||
const minutes = healthy && m.wasHealthy && contiguous ? gap / 60_000 : 0;
|
||||
if (!healthy && m.wasHealthy === true) m.outages++;
|
||||
for (const r of plan.requirements) {
|
||||
const value = m.values[r.id] ?? 0;
|
||||
const populated = players >= Math.max(1, r.minPlayers) && (m.lastPlayers ?? 0) >= Math.max(1, r.minPlayers);
|
||||
if (r.kind === 'populated-minutes' && populated) m.values[r.id] = value + minutes;
|
||||
if (r.kind === 'player-hours' && populated) m.values[r.id] = value + minutes * Math.min(players, m.lastPlayers ?? 0) / 60;
|
||||
if (r.kind === 'population' && healthy) m.values[r.id] = Math.max(value, players);
|
||||
if (r.kind === 'healthy-minutes') m.values[r.id] = healthy && contiguous && m.wasHealthy ? value + minutes : 0;
|
||||
if (r.kind === 'sustained-population') {
|
||||
m.streaks[r.id] = healthy && populated && contiguous ? (m.streaks[r.id] ?? 0) + minutes : 0;
|
||||
m.values[r.id] = Math.max(value, m.streaks[r.id]);
|
||||
}
|
||||
}
|
||||
m.lastSample = at; m.wasHealthy = healthy; m.lastPlayers = players;
|
||||
}
|
||||
|
||||
export function evaluateRequirements(plan: BetaPlan, metrics: TestMetrics, phase: number): RequirementProgress[] {
|
||||
return plan.requirements.map(r => {
|
||||
const budget = ['failure-budget', 'restart-budget', 'clean-logs'].includes(r.kind);
|
||||
const current = r.kind === 'failure-budget' ? metrics.outages : r.kind === 'clean-logs' ? metrics.logErrors
|
||||
: r.kind === 'restart-budget' ? metrics.restarts
|
||||
: r.kind === 'manual' ? Number(!!metrics.manual[r.id]) : r.kind === 'recovery' ? phase : metrics.values[r.id] ?? 0;
|
||||
const available = (r.kind !== 'clean-logs' || metrics.logAvailable === true) && (r.kind !== 'restart-budget' || !!metrics.bootId);
|
||||
return { id: r.id, label: r.label, current, target: r.target, passed: available && (budget ? current <= r.target : current >= r.target),
|
||||
failed: budget && current > r.target,
|
||||
detail: r.kind === 'manual' ? metrics.manual[r.id] ?? 'Awaiting tester sign-off'
|
||||
: !available ? r.kind === 'restart-budget' ? 'Waiting for boot telemetry' : 'Waiting for a readable console log' : budget ? `${current} / ${r.target} allowed; monitored until promotion`
|
||||
: `${Math.round(current * 100) / 100} / ${r.target}` };
|
||||
});
|
||||
}
|
||||
|
||||
export function coveragePassed(plan: BetaPlan, servers: Array<{ passed: boolean; game: string | null; map: string | null }>): boolean {
|
||||
if (!servers.length) return false;
|
||||
if (plan.scope === 'all') return servers.every(s => s.passed);
|
||||
if (plan.scope === 'quorum') return servers.filter(s => s.passed).length >= plan.quorum;
|
||||
const key = (s: typeof servers[number]) => plan.scope === 'game' ? s.game : plan.scope === 'map' ? s.map
|
||||
: s.game && s.map ? `${s.game}/${s.map}` : null;
|
||||
const groups = new Set(servers.map(key));
|
||||
return !groups.has(null) && [...groups].every(group => servers.some(s => s.passed && key(s) === group));
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { open, stat } from 'node:fs/promises';
|
||||
import { db } from '../db/index.js';
|
||||
import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan.js';
|
||||
import { applySample, applySignal, emptyMetrics, evaluateRequirements, type TestMetrics, type TestSignal } from './beta-evaluation.js';
|
||||
import { gameLogKey, resolveLogPath, type LogKey } from './logs.js';
|
||||
|
||||
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string; cursors: string }
|
||||
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
|
||||
interface Cursor { position: number; identity: string; partial: string; anchor?: string; generation?: string; session?: string; joined?: boolean; started?: number }
|
||||
const nowIso = () => new Date().toISOString();
|
||||
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
|
||||
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
|
||||
export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); }
|
||||
export function setOutcome(id: number, outcome: string, reason: string | null): void {
|
||||
const previous = storedPlan(id);
|
||||
if (!previous || (previous.outcome === outcome && previous.reason === reason)) return;
|
||||
db().prepare('UPDATE beta_test_plans SET outcome = ?, reason = ? WHERE beta_id = ?').run(outcome, reason, id);
|
||||
activity(id, `Test plan ${outcome}${reason ? `: ${reason}` : ''}`);
|
||||
}
|
||||
export function readMetrics(run: TestRun): TestMetrics { return { ...emptyMetrics(), ...JSON.parse(run.metrics) }; }
|
||||
export function saveMetrics(run: TestRun, metrics: TestMetrics): void { db().prepare('UPDATE beta_test_runs SET metrics = ? WHERE id = ?').run(JSON.stringify(metrics), run.id); }
|
||||
export function evidence(run: TestRun, eventId: string, kind: string, payload: unknown): boolean {
|
||||
return db().prepare('INSERT OR IGNORE INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(run.id, eventId, nowIso(), kind, JSON.stringify(payload)).changes > 0;
|
||||
}
|
||||
|
||||
/** Snapshot log offsets BEFORE restart: pre-beta log entries can never qualify. */
|
||||
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
|
||||
const cursors: Record<string, Cursor> = {};
|
||||
for (const key of [gameLogKey(server), 'console'] as LogKey[]) {
|
||||
const path = resolveLogPath(server, key);
|
||||
const info = path ? await stat(path).catch(() => null) : null;
|
||||
let anchor = '';
|
||||
if (path && info?.size) {
|
||||
const file = await open(path, 'r');
|
||||
try {
|
||||
const buffer = Buffer.alloc(Math.min(info.size, 128));
|
||||
await file.read(buffer, 0, buffer.length, info.size - buffer.length);
|
||||
anchor = buffer.toString('base64');
|
||||
} finally { await file.close(); }
|
||||
}
|
||||
cursors[key] = { position: info?.size ?? 0, identity: info ? `${info.dev}:${info.ino}` : '', partial: '', anchor, generation: randomUUID() };
|
||||
}
|
||||
const id = randomUUID();
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
|
||||
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, cursors) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
||||
.run(id, betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), JSON.stringify(cursors));
|
||||
})();
|
||||
return currentRun(betaId, server)!;
|
||||
}
|
||||
|
||||
export function acceptTestSignal(server: string, runId: string, eventId: string, signal: TestSignal): { accepted: boolean; reason?: string } {
|
||||
return db().transaction(() => {
|
||||
const run = db().prepare(`SELECT r.* FROM beta_test_runs r JOIN betas b ON b.id = r.beta_id
|
||||
JOIN beta_test_plans p ON p.beta_id = b.id WHERE r.id = ? AND r.server = ? AND r.retired_at IS NULL
|
||||
AND b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed')`).get(runId, server) as TestRun | undefined;
|
||||
if (!run) return { accepted: false, reason: 'Run is inactive or superseded; obtain the current run at process startup' };
|
||||
if (db().prepare('SELECT 1 FROM beta_test_evidence WHERE run_id = ? AND event_id = ?').get(runId, `telemetry:${eventId}`)) return { accepted: false, reason: 'Duplicate event' };
|
||||
const plan = parseBetaPlan(JSON.parse(storedPlan(run.beta_id)!.plan));
|
||||
const metrics = readMetrics(run);
|
||||
if (!applySignal(plan, metrics, signal, Date.now())) return { accepted: false, reason: 'Duplicate, out-of-order, closed, or unknown session/boot' };
|
||||
evidence(run, `telemetry:${eventId}`, signal.event, signal);
|
||||
saveMetrics(run, metrics);
|
||||
return { accepted: true };
|
||||
})();
|
||||
}
|
||||
|
||||
export function parseTestSignal(body: Record<string, unknown>): TestSignal {
|
||||
if (!['boot', 'session_start', 'round_change', 'match_end', 'feature'].includes(String(body.event))) throw new Error('Unknown telemetry event');
|
||||
for (const key of ['runId', 'eventId', 'bootId'] as const) {
|
||||
if (typeof body[key] !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body[key] as string)) throw new Error(`${key} is required (1–128 identifier characters)`);
|
||||
}
|
||||
if (body.event !== 'boot' && (typeof body.sessionId !== 'string' || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(body.sessionId))) throw new Error('sessionId is required');
|
||||
for (const key of ['sequence', 'round', 'players']) if (!Number.isSafeInteger(body[key]) || (body[key] as number) < 0) throw new Error(`${key} must be a nonnegative integer`);
|
||||
if (body.event === 'feature' && (typeof body.name !== 'string' || !/^[a-zA-Z0-9_.:-]{1,120}$/.test(body.name))) throw new Error('Feature name is required');
|
||||
return { event: body.event as TestSignal['event'], bootId: body.bootId as string, sessionId: body.sessionId as string ?? '',
|
||||
sequence: body.sequence as number, round: body.round as number, players: body.players as number, name: body.name as string ?? '' };
|
||||
}
|
||||
|
||||
/** Bounded incremental reads, with persistent byte cursors and partial lines. */
|
||||
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
|
||||
const cursors = JSON.parse(run.cursors) as Record<string, Cursor>;
|
||||
const observed: Array<{ key: LogKey; line: string; id: string }> = [];
|
||||
let logAvailable = false;
|
||||
for (const key of [gameLogKey(run.server), 'console'] as LogKey[]) {
|
||||
const path = resolveLogPath(run.server, key);
|
||||
const file = path ? await open(path, 'r').catch(() => null) : null;
|
||||
if (!file) continue;
|
||||
try {
|
||||
const info = await file.stat();
|
||||
if (key === 'console') logAvailable = true;
|
||||
const identity = `${info.dev}:${info.ino}`;
|
||||
let cursor = cursors[key] ?? { position: 0, identity, partial: '' };
|
||||
let anchorChanged = false;
|
||||
if (cursor.anchor && info.size >= cursor.position) {
|
||||
const expected = Buffer.from(cursor.anchor, 'base64');
|
||||
const actual = Buffer.alloc(expected.length);
|
||||
await file.read(actual, 0, actual.length, cursor.position - actual.length);
|
||||
anchorChanged = !actual.equals(expected);
|
||||
}
|
||||
if (cursor.identity !== identity || info.size < cursor.position || anchorChanged) {
|
||||
cursor = { position: 0, identity, partial: '', generation: randomUUID() };
|
||||
observed.push({ key, line: '', id: `rotation:${key}:${randomUUID()}` });
|
||||
}
|
||||
if (info.size - cursor.position > 2_000_000) {
|
||||
setOutcome(run.beta_id, 'inconclusive', `${run.server}: log reader fell behind; evidence is incomplete`);
|
||||
return;
|
||||
}
|
||||
const buffer = Buffer.alloc(info.size - cursor.position);
|
||||
const { bytesRead } = await file.read(buffer, 0, buffer.length, cursor.position);
|
||||
const combined = Buffer.concat([Buffer.from(cursor.partial, 'base64'), buffer.subarray(0, bytesRead)]);
|
||||
let start = 0;
|
||||
for (let i = 0; i < combined.length; i++) if (combined[i] === 10) {
|
||||
observed.push({ key, line: combined.subarray(start, i).toString('utf8'), id: `${key}:${identity}:${cursor.generation ?? ''}:${cursor.position + i - Buffer.from(cursor.partial, 'base64').length}` });
|
||||
start = i + 1;
|
||||
}
|
||||
cursor.position += bytesRead;
|
||||
const anchor = Buffer.alloc(Math.min(cursor.position, 128));
|
||||
await file.read(anchor, 0, anchor.length, cursor.position - anchor.length);
|
||||
cursor.anchor = anchor.toString('base64');
|
||||
cursor.partial = combined.subarray(start).toString('base64');
|
||||
if (cursor.partial.length > 100_000) cursor.partial = ''; // malformed giant line cannot grow forever
|
||||
cursor.identity = identity;
|
||||
cursors[key] = cursor;
|
||||
} finally { await file.close(); }
|
||||
}
|
||||
db().transaction(() => {
|
||||
const fresh = currentRun(run.beta_id, run.server);
|
||||
if (!fresh || fresh.id !== run.id) return;
|
||||
const m = readMetrics(fresh);
|
||||
m.logAvailable = logAvailable;
|
||||
for (const entry of observed) {
|
||||
const cursor = cursors[entry.key];
|
||||
if (!entry.line) { evidence(run, entry.id, 'log-rotation', { log: entry.key }); continue; }
|
||||
if (entry.key === 'console') {
|
||||
if (/script (?:runtime )?error|unhandled exception|script exception|fatal error/i.test(entry.line) && evidence(run, entry.id, 'script-error', { line: entry.line.slice(0, 1000) })) m.logErrors++;
|
||||
continue;
|
||||
}
|
||||
if (/InitGame:/.test(entry.line)) { cursor.session = entry.id; cursor.joined = false; cursor.started = undefined; }
|
||||
if (cursor.session && /\sJ;[^;]*;\d+;/.test(entry.line) && !cursor.joined) { cursor.joined = true; cursor.started = Date.now(); }
|
||||
if (/ExitLevel:\s*executed/.test(entry.line) && cursor.session && cursor.joined) {
|
||||
// Unqualified match counts use logs. Round/population-qualified counts use session telemetry exclusively.
|
||||
if (evidence(run, `match:${cursor.session}`, 'completed-match', { session: cursor.session, line: entry.line, minutes: (Date.now() - (cursor.started ?? Date.now())) / 60_000 })) {
|
||||
for (const r of plan.requirements) if (r.kind === 'matches' && r.minRound === 0 && r.minPlayers <= 1
|
||||
&& Date.now() - (cursor.started ?? Date.now()) >= r.minMinutes * 60_000) m.values[r.id] = (m.values[r.id] ?? 0) + 1;
|
||||
}
|
||||
cursor.joined = false;
|
||||
}
|
||||
if (/ShutdownGame:/.test(entry.line)) cursor.joined = false;
|
||||
}
|
||||
const server = db().prepare('SELECT last_checked_at, last_seen_at, last_players, last_error, container_state FROM servers WHERE name = ?').get(run.server) as {
|
||||
last_checked_at: string | null; last_seen_at: string | null; last_players: number | null; last_error: string | null; container_state: string | null };
|
||||
const sample = server.last_checked_at ? Date.parse(server.last_checked_at) : 0;
|
||||
const freshSample = sample >= Date.parse(run.observing_at ?? run.created_at) && Date.now() - sample <= 60_000;
|
||||
const healthy = freshSample && !server.last_error && server.container_state === 'running' && server.last_seen_at === server.last_checked_at;
|
||||
const at = freshSample && server.container_state === 'running' ? sample : Date.now();
|
||||
// Startup requires fresh status; don't count an absent first observation as an outage.
|
||||
const before = m.wasHealthy;
|
||||
const lastSample = m.lastSample;
|
||||
applySample(plan, m, at, healthy, healthy ? server.last_players ?? 0 : 0);
|
||||
if (freshSample && sample !== lastSample) evidence(run, `sample:${sample}`, 'status-sample', { healthy, players: server.last_players, checkedAt: server.last_checked_at });
|
||||
if (before !== m.wasHealthy) evidence(run, `health:${at}`, 'health', { healthy, players: server.last_players, reason: server.last_error });
|
||||
saveMetrics(run, m);
|
||||
db().prepare('UPDATE beta_test_runs SET cursors = ? WHERE id = ?').run(JSON.stringify(cursors), run.id);
|
||||
})();
|
||||
}
|
||||
|
||||
export function testView(betaId: number): BetaTestView | null {
|
||||
const stored = storedPlan(betaId);
|
||||
if (!stored) return null;
|
||||
const plan = parseBetaPlan(JSON.parse(stored.plan));
|
||||
const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL ORDER BY server').all(betaId) as TestRun[];
|
||||
return { plan, outcome: stored.outcome, reason: stored.reason,
|
||||
servers: runs.map(run => {
|
||||
const progress = evaluateRequirements(plan, readMetrics(run), run.phase);
|
||||
return { server: run.server, runId: run.id, phase: run.phase, passed: progress.every(r => r.passed), progress };
|
||||
}),
|
||||
evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e
|
||||
JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
|
||||
}
|
||||
@@ -1,3 +1,14 @@
|
||||
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
|
||||
import { storedPlan } from './beta-tests.js';
|
||||
|
||||
const operations = new Set<number>();
|
||||
export const betaOperationInFlight = (id: number) => operations.has(id);
|
||||
async function exclusive<T>(id: number, action: () => Promise<T>): Promise<T> {
|
||||
if (operations.has(id)) throw new BetaError('A beta operation is already running', 'invalid-state');
|
||||
operations.add(id);
|
||||
try { return await action(); } finally { operations.delete(id); }
|
||||
}
|
||||
|
||||
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { dirname, join, posix } from 'node:path';
|
||||
@@ -229,7 +240,7 @@ export function findRunningBetaForScript(scriptId: number): BetaRow | undefined
|
||||
.prepare(
|
||||
`SELECT b.* FROM betas b
|
||||
JOIN beta_scripts bs ON bs.beta_id = b.id
|
||||
WHERE bs.script_id = ? AND b.state = 'running'
|
||||
WHERE bs.script_id = ? AND b.state IN ('running','ready-promote')
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(scriptId) as BetaRow | undefined;
|
||||
@@ -245,6 +256,7 @@ export interface OpenBetaScript {
|
||||
export interface OpenBetaInput {
|
||||
scripts: OpenBetaScript[];
|
||||
servers: string[];
|
||||
testPlan?: BetaPlan;
|
||||
waitForHealth: boolean;
|
||||
waitForMatch: boolean;
|
||||
notes?: string;
|
||||
@@ -257,6 +269,9 @@ export interface OpenBetaResult {
|
||||
}
|
||||
|
||||
export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
const plan = input.testPlan ? parseBetaPlan(input.testPlan) : null;
|
||||
if (new Set(input.servers).size !== input.servers.length) throw new BetaError("Select each server only once", "invalid-state");
|
||||
if (plan?.scope === "quorum" && plan.quorum > input.servers.length) throw new BetaError("Quorum exceeds selected servers", "invalid-state");
|
||||
if (!input.scripts || input.scripts.length === 0) {
|
||||
throw new BetaError('at least one script is required', 'no-scripts');
|
||||
}
|
||||
@@ -319,6 +334,11 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
serverRows.push({ name, game: row.game, map: row.map });
|
||||
}
|
||||
|
||||
if (plan && ['map', 'game', 'map-game'].includes(plan.scope)) {
|
||||
const missing = serverRows.some(s => (plan.scope !== 'map' && !s.game) || (plan.scope !== 'game' && !s.map));
|
||||
if (missing) throw new BetaError('Coverage requires game/map metadata on every selected server', 'invalid-state');
|
||||
}
|
||||
|
||||
// Per-script scope validation. Aggregate all violations into one error.
|
||||
const outOfScope: Array<{ scriptId: number; scope: string; servers: string[] }> = [];
|
||||
for (const { script } of resolved) {
|
||||
@@ -387,11 +407,16 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
input.notes ?? null
|
||||
);
|
||||
betaId = Number(res.lastInsertRowid);
|
||||
if (plan) db().prepare('INSERT INTO beta_test_plans (beta_id, plan) VALUES (?, ?)').run(betaId, JSON.stringify(plan));
|
||||
for (const r of resolved) {
|
||||
insertBetaScript.run(betaId, r.script.id, r.version.id, r.script.kind);
|
||||
}
|
||||
for (const name of input.servers) {
|
||||
insertBetaServer.run(betaId, name, nowIso());
|
||||
if (plan) {
|
||||
const selected = serverRows.find(s => s.name === name)!;
|
||||
db().prepare('INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (?, ?, ?, ?)').run(betaId, name, selected.game, selected.map);
|
||||
}
|
||||
for (const r of resolved) {
|
||||
const prior = getServerScript(name, r.script.target_rel_path);
|
||||
insertBetaServerScript.run(
|
||||
@@ -405,6 +430,30 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
}
|
||||
})();
|
||||
|
||||
if (plan?.waveSize) db().prepare("UPDATE beta_servers SET stage = 'queued' WHERE beta_id = ?").run(betaId);
|
||||
await deployBetaServers(betaId, plan?.waveSize ? input.servers.slice(0, plan.waveSize) : input.servers);
|
||||
|
||||
emit({
|
||||
type: 'beta-opened',
|
||||
betaId,
|
||||
scriptIds: resolved.map((r) => r.script.id),
|
||||
servers: input.servers
|
||||
});
|
||||
|
||||
return {
|
||||
betaId,
|
||||
servers: input.servers,
|
||||
scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id }))
|
||||
};
|
||||
}
|
||||
|
||||
/** Apply a reserved wave only when its predecessors have passed. */
|
||||
export async function deployBetaServers(betaId: number, names: string[]): Promise<void> {
|
||||
return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
|
||||
}
|
||||
async function deployBetaServersInternal(betaId: number, names: string[]): Promise<void> {
|
||||
const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! }));
|
||||
const serverRows = names.map(name => getServer(name)!);
|
||||
// Split resolved artifacts into non-collapsing (each writes its own file)
|
||||
// and collapsing (config/baseline overlay onto one shared dedicated_sp.cfg).
|
||||
const nonCollapsing = resolved.filter((r) => !isCollapsingKind(r.script.kind));
|
||||
@@ -434,7 +483,9 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
WHERE beta_id = ? AND server = ?`
|
||||
);
|
||||
|
||||
for (const name of input.servers) {
|
||||
for (const name of names) {
|
||||
db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?")
|
||||
.run(nowIso(), betaId, name);
|
||||
const serverRow = serverRows.find((r) => r.name === name)!;
|
||||
let firstErr: string | null = null;
|
||||
// 1. Non-collapsing artifacts: script/plugin — write directly to the
|
||||
@@ -456,7 +507,9 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
sha256Buffer(priorBytes),
|
||||
priorBytes.byteLength
|
||||
);
|
||||
db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id);
|
||||
}
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path);
|
||||
await writeFile(targetPath, version.content);
|
||||
recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`);
|
||||
} catch (err) {
|
||||
@@ -494,6 +547,7 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
);
|
||||
}
|
||||
const composedBuf = Buffer.from(composedResult.text, 'utf8');
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel);
|
||||
await writeFile(targetPath, composedBuf);
|
||||
recordScript(
|
||||
name,
|
||||
@@ -519,18 +573,6 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
}
|
||||
}
|
||||
|
||||
emit({
|
||||
type: 'beta-opened',
|
||||
betaId,
|
||||
scriptIds: resolved.map((r) => r.script.id),
|
||||
servers: input.servers
|
||||
});
|
||||
|
||||
return {
|
||||
betaId,
|
||||
servers: input.servers,
|
||||
scripts: resolved.map((r) => ({ scriptId: r.script.id, scriptVersionId: r.version.id }))
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- rollback ----------
|
||||
@@ -543,6 +585,9 @@ export interface RollbackResult {
|
||||
}
|
||||
|
||||
export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
|
||||
return exclusive(betaId, () => rollbackBetaInternal(betaId));
|
||||
}
|
||||
async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
|
||||
const beta = getBeta(betaId);
|
||||
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
|
||||
if (beta.state !== 'running' && beta.state !== 'ready-promote') {
|
||||
@@ -570,8 +615,10 @@ export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
|
||||
);
|
||||
|
||||
for (const bs of servers) {
|
||||
if (!bs.deployed_at && bs.stage === 'queued') continue;
|
||||
for (const { script } of bundle) {
|
||||
if (isCollapsingKind(script.kind)) continue; // handled once per server below
|
||||
if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue;
|
||||
const dest = destOfKind(bs.server, script.kind);
|
||||
const targetPath = join(dest, script.target_rel_path);
|
||||
|
||||
@@ -601,7 +648,7 @@ export async function rollbackBeta(betaId: number): Promise<RollbackResult> {
|
||||
}
|
||||
|
||||
// Collapsing config layer: single restore for the shared cfg file.
|
||||
if (hasCollapsing) {
|
||||
if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) {
|
||||
const dest = destOfKind(bs.server, 'config');
|
||||
const targetRel = configFileOf(bs.server);
|
||||
const targetPath = join(dest, targetRel);
|
||||
@@ -681,6 +728,15 @@ function promoteDeploymentId(betaId: number): string {
|
||||
}
|
||||
|
||||
export async function promoteBeta(betaId: number): Promise<PromoteResult> {
|
||||
if (storedPlan(betaId)) {
|
||||
const { plannedBetaTick } = await import('../pipeline/beta-plan-runner.js');
|
||||
await plannedBetaTick();
|
||||
}
|
||||
return exclusive(betaId, () => promoteBetaInternal(betaId));
|
||||
}
|
||||
async function promoteBetaInternal(betaId: number): Promise<PromoteResult> {
|
||||
const test = storedPlan(betaId);
|
||||
if (test && test.outcome !== 'passed') throw new BetaError('Test plan has not passed', 'not-ready-promote');
|
||||
const beta = getBeta(betaId);
|
||||
if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
|
||||
if (beta.state !== 'ready-promote') {
|
||||
|
||||
@@ -316,7 +316,7 @@ export function deleteVersion(versionId: number): void {
|
||||
.prepare(
|
||||
`SELECT b.id FROM betas b
|
||||
JOIN beta_scripts bs ON bs.beta_id = b.id
|
||||
WHERE bs.script_version_id = ? AND b.state = 'running' LIMIT 1`
|
||||
WHERE bs.script_version_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
|
||||
)
|
||||
.get(versionId) as { id: number } | undefined;
|
||||
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this version`, 'in-use-beta');
|
||||
@@ -334,7 +334,7 @@ export function deleteScript(scriptId: number): void {
|
||||
.prepare(
|
||||
`SELECT b.id FROM betas b
|
||||
JOIN beta_scripts bs ON bs.beta_id = b.id
|
||||
WHERE bs.script_id = ? AND b.state = 'running' LIMIT 1`
|
||||
WHERE bs.script_id = ? AND b.state IN ('running','ready-promote') LIMIT 1`
|
||||
)
|
||||
.get(scriptId) as { id: number } | undefined;
|
||||
if (beta) throw new ScriptError(`beta ${beta.id} is still running against this script`, 'has-active-beta');
|
||||
|
||||
@@ -204,7 +204,7 @@ export function heldByBeta(server: string): Map<string, number> {
|
||||
JOIN betas b ON b.id = bsv.beta_id
|
||||
JOIN beta_scripts bs ON bs.beta_id = b.id
|
||||
JOIN scripts s ON s.id = bs.script_id
|
||||
WHERE bsv.server = ? AND b.state = 'running'`
|
||||
WHERE bsv.server = ? AND b.state IN ('running','ready-promote')`
|
||||
)
|
||||
.all(server) as Array<{ target: string; beta_id: number }>;
|
||||
const map = new Map<string, number>();
|
||||
@@ -223,7 +223,7 @@ export function ownerOf(server: string, target: string): number | null {
|
||||
JOIN betas b ON b.id = bsv.beta_id
|
||||
JOIN beta_scripts bs ON bs.beta_id = b.id
|
||||
JOIN scripts s ON s.id = bs.script_id
|
||||
WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state = 'running'
|
||||
WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state IN ('running','ready-promote')
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(server, target) as { beta_id: number } | undefined;
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
|
||||
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
|
||||
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
|
||||
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
|
||||
import { emit, type BetaStage } from './bus.js';
|
||||
import { restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
|
||||
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
|
||||
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
|
||||
.run(next, error, new Date().toISOString(), id, server);
|
||||
emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
|
||||
}
|
||||
|
||||
export function betaState(id: number, ready: boolean): void {
|
||||
const state = ready ? 'ready-promote' : 'running';
|
||||
const result = db().prepare("UPDATE betas SET state = ? WHERE id = ? AND state IN ('running','ready-promote') AND state <> ?").run(state, id, state);
|
||||
if (result.changes) emit({ type: 'beta-state', betaId: id, state });
|
||||
// A revoked pass must not leave this version labelled tested unless another beta still proves it.
|
||||
db().prepare(`UPDATE script_versions SET state = ? WHERE id IN (SELECT script_version_id FROM beta_scripts WHERE beta_id = ?)
|
||||
AND id NOT IN (SELECT current_version_id FROM scripts WHERE current_version_id IS NOT NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM beta_scripts bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.script_version_id = script_versions.id AND b.id <> ? AND b.state IN ('ready-promote','promoted'))`)
|
||||
.run(ready ? 'tested' : 'untested', id, id);
|
||||
}
|
||||
|
||||
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
|
||||
const servers = listBetaServers(id).filter(s => s.deployed_at);
|
||||
const result = await rollbackBeta(id);
|
||||
if (plan.rollbackRestart === 'when-empty' && servers.length) {
|
||||
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
|
||||
db().transaction(() => {
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
|
||||
.run(deploymentId, new Date().toISOString(), id);
|
||||
for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
|
||||
})();
|
||||
activity(id, 'Restored files; rolling restarts will wait for empty servers');
|
||||
} else activity(id, 'Restored files; manual restarts are required to load them');
|
||||
if (result.warnings.length) activity(id, result.warnings.join('; '));
|
||||
}
|
||||
|
||||
async function fail(id: number, plan: BetaPlan, reason: string): Promise<void> {
|
||||
betaState(id, false);
|
||||
setOutcome(id, plan.failureAction === 'pause' ? 'paused' : 'failed', reason);
|
||||
if (plan.failureAction === 'rollback') {
|
||||
try { await rollbackTestPlan(id, plan); }
|
||||
catch (error) { setOutcome(id, 'failed', `Automatic rollback failed: ${(error as Error).message}`); }
|
||||
}
|
||||
}
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
export function plannedBetaTick(): Promise<void> {
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = tick().finally(() => { inFlight = null; });
|
||||
return inFlight;
|
||||
}
|
||||
async function tick(): Promise<void> {
|
||||
const plans = db().prepare(`SELECT p.* FROM beta_test_plans p JOIN betas b ON b.id = p.beta_id
|
||||
WHERE b.state IN ('running','ready-promote') AND p.outcome IN ('active','passed') ORDER BY p.beta_id`).all() as StoredPlan[];
|
||||
for (const stored of plans) {
|
||||
if (betaOperationInFlight(stored.beta_id)) continue;
|
||||
try { await advancePlan(stored); }
|
||||
catch (error) {
|
||||
const reason = `Test runner: ${(error as Error).message}`;
|
||||
try { await fail(stored.beta_id, parseBetaPlan(JSON.parse(stored.plan)), reason); }
|
||||
catch { betaState(stored.beta_id, false); setOutcome(stored.beta_id, 'failed', reason); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
const id = stored.beta_id;
|
||||
const plan = parseBetaPlan(JSON.parse(stored.plan));
|
||||
const beta = getBeta(id)!;
|
||||
const servers = listBetaServers(id);
|
||||
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
|
||||
betaState(id, false);
|
||||
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
|
||||
return;
|
||||
}
|
||||
if (!komodoConfigured() || !rconConfigured()) {
|
||||
betaState(id, false);
|
||||
setOutcome(id, 'active', 'Waiting for Komodo and RCON configuration');
|
||||
return;
|
||||
}
|
||||
for (const server of servers) {
|
||||
const freshPlan = storedPlan(id)!;
|
||||
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
|
||||
if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
|
||||
if (server.stage === 'deploying' || server.stage === 'restarting') {
|
||||
stage(id, server.server, 'failed', `${server.stage === 'deploying' ? 'deploy: ' : ''}Operation interrupted; review the server before starting a fresh attempt`);
|
||||
await fail(id, plan, `${server.server}: interrupted ${server.stage}`); return;
|
||||
}
|
||||
if (server.stage === 'queued') continue;
|
||||
if (server.stage === 'awaiting-empty') {
|
||||
const waitingRun = currentRun(id, server.server);
|
||||
if (waitingRun?.observing_at) {
|
||||
await observeRun(waitingRun, plan);
|
||||
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
|
||||
const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed);
|
||||
if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
|
||||
}
|
||||
if (rolloutWaitReason(server.server)) continue;
|
||||
if (await rconGetPlayerCount(server.server) !== 0) continue;
|
||||
const oldRun = currentRun(id, server.server);
|
||||
const phase = oldRun ? oldRun.phase + 1 : 0;
|
||||
// Fail closed after interrupted restart requests; never silently reuse earlier evidence.
|
||||
stage(id, server.server, 'restarting');
|
||||
const run = await prepareRun(id, server.server, phase);
|
||||
if (oldRun) {
|
||||
const previous = readMetrics(oldRun);
|
||||
const metrics = readMetrics(run);
|
||||
metrics.outages = previous.outages; metrics.logErrors = previous.logErrors; metrics.restarts = previous.restarts;
|
||||
saveMetrics(run, metrics);
|
||||
}
|
||||
const restart = await restartInRollout(server.server);
|
||||
if (restart.kind === 'queued') {
|
||||
db().prepare('DELETE FROM beta_test_runs WHERE id = ?').run(run.id);
|
||||
if (oldRun) db().prepare('UPDATE beta_test_runs SET retired_at = NULL WHERE id = ?').run(oldRun.id);
|
||||
stage(id, server.server, 'awaiting-empty', restart.message); continue;
|
||||
}
|
||||
if (restart.kind === 'failed') { stage(id, server.server, 'failed', restart.message); await fail(id, plan, restart.message); return; }
|
||||
db().prepare('UPDATE beta_servers SET restarted_at = ?, healthy_at = NULL WHERE beta_id = ? AND server = ?')
|
||||
.run(new Date().toISOString(), id, server.server);
|
||||
stage(id, server.server, 'awaiting-health');
|
||||
continue;
|
||||
}
|
||||
if (server.stage === 'awaiting-health') {
|
||||
const health = rolloutHealth(server.server);
|
||||
if (!health.ok) continue;
|
||||
const now = new Date().toISOString();
|
||||
db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server);
|
||||
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server);
|
||||
stage(id, server.server, 'observing');
|
||||
}
|
||||
const run = currentRun(id, server.server);
|
||||
if (!run?.observing_at) continue;
|
||||
await observeRun(run, plan);
|
||||
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
|
||||
const metrics = readMetrics(currentRun(id, server.server)!);
|
||||
const progress = evaluateRequirements(plan, metrics, run.phase);
|
||||
const broken = progress.find(p => p.failed);
|
||||
if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
|
||||
const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target));
|
||||
if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) {
|
||||
stage(id, server.server, 'awaiting-empty');
|
||||
activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`);
|
||||
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
|
||||
}
|
||||
const updated = listBetaServers(id);
|
||||
const queued = updated.filter(s => s.stage === 'queued');
|
||||
if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
|
||||
const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);
|
||||
activity(id, `Previous wave passed; deploying ${wave.join(', ')}`);
|
||||
await deployBetaServers(id, wave);
|
||||
betaState(id, false);
|
||||
return;
|
||||
}
|
||||
const registry = db().prepare('SELECT server AS name, game, map FROM beta_test_coverage WHERE beta_id = ?').all(id) as Array<{ name: string; game: string | null; map: string | null }>;
|
||||
// All waves must be activated, even when the final coverage rule is a quorum.
|
||||
const ready = !queued.length && updated.every(s => !!currentRun(id, s.server)?.observing_at)
|
||||
&& coveragePassed(plan, updated.map(s => ({ ...registry.find(r => r.name === s.server)!, passed: s.stage === 'ready' })));
|
||||
betaState(id, ready);
|
||||
setOutcome(id, ready ? 'passed' : 'active', null);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { scanForMatchEnd } from '../fleet/match-log.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
import { plannedBetaTick } from './beta-plan-runner.js';
|
||||
|
||||
interface BetaServerActiveRow {
|
||||
beta_id: number;
|
||||
@@ -93,8 +94,9 @@ export function betaTick(): Promise<{ processed: number }> {
|
||||
}
|
||||
|
||||
async function runBetaTick(): Promise<{ processed: number }> {
|
||||
if (integrationsOn()) await refreshRestartRollout();
|
||||
await plannedBetaTick();
|
||||
if (!integrationsOn()) return { processed: 0 };
|
||||
await refreshRestartRollout();
|
||||
|
||||
const rows = db()
|
||||
.prepare(
|
||||
@@ -104,6 +106,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
|
||||
FROM beta_servers bs
|
||||
JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE b.state = 'running'
|
||||
AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id)
|
||||
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
|
||||
ORDER BY b.id, bs.server`
|
||||
)
|
||||
|
||||
@@ -26,6 +26,9 @@ export interface ServerStatusEntry {
|
||||
}
|
||||
|
||||
export type BetaStage =
|
||||
| 'queued'
|
||||
| 'observing'
|
||||
| 'inconclusive'
|
||||
| 'deploying'
|
||||
| 'awaiting-empty'
|
||||
| 'restarting'
|
||||
|
||||
Reference in New Issue
Block a user