Allow revising active betas with fresh tests and history

This commit is contained in:
2026-10-05 21:02:58 -04:00
parent 9d4ff992a4
commit f8d0ba3936
21 changed files with 713 additions and 41 deletions
+35 -3
View File
@@ -197,8 +197,8 @@ Set wave size to 1 for a canary, a larger number for batches, or 0 for all
selected servers. Later waves receive files after the previous waves pass. selected servers. Later waves receive files after the previous waves pass.
Health-outage budget violations on any tested server block promotion. Health-outage budget violations on any tested server block promotion.
The test window starts when the beta opens, including time waiting for empty The test window starts when the beta opens or a revision is submitted, including
servers. Missing evidence at the deadline produces **inconclusive**. time waiting for empty servers. Missing evidence at the deadline produces **inconclusive**.
Health monitoring continues after a pass until promotion. Health monitoring continues after a pass until promotion.
For failures, select hold as failed, pause for investigation, or automatic For failures, select hold as failed, pause for investigation, or automatic
@@ -208,7 +208,7 @@ The manual option holds those changes in a rollback campaign until you select
**Resume when empty**. Promotion is always manual. **Resume when empty**. Promotion is always manual.
Pause holds evaluation and wave advancement. Resume retains completed progress and the Pause holds evaluation and wave advancement. Resume retains completed progress and the
original deadline, but starts new timing intervals. A fresh attempt resets current revision's deadline, but starts new timing intervals. A fresh attempt resets
progress and restarts deployed servers when empty. Recovery cycles repeat progress and restarts deployed servers when empty. Recovery cycles repeat
gameplay milestones and checklist sign-offs while carrying health outages forward. gameplay milestones and checklist sign-offs while carrying health outages forward.
@@ -217,6 +217,32 @@ recent 200 evidence entries. On upgrading from the old reporter protocol,
active progress resets. Unsupported checks in existing plans and saved presets active progress resets. Unsupported checks in existing plans and saved presets
become manual review items; they cannot silently pass. become manual review items; they cannot silently pass.
## Updating an active beta
Choose **Update beta** on the beta detail page, upload replacements for existing
bundle files, and review the changes and selected servers before choosing
**Deploy revision**. Text files show a diff when both files are small enough;
binary and large files show their checksums. Files must actually change.
The beta keeps its ID, servers, scope and test policy. Each revision creates new
file versions, clears current test progress and checklist sign-offs, repeats the
configured waves, and starts a fresh testing window at submission. Files install
and servers restart through the normal empty-server and capacity gates. Promotion
requires the new revision to pass. Pause and retry retain the current revision's
deadline; they do not create a revision.
Revision history retains downloadable bundle files and previous test results,
including the last 200 evidence entries shown at replacement. All underlying
attempts and evidence remain in SQLite, separated by revision. Rejecting the beta
restores the original pre-beta files, even after multiple revisions or while a
new revision is waiting to deploy. Files originally absent are removed.
Updates are available until promotion or rollback, including paused, failed and
expired tests. An activation already in progress must settle first; interrupted
file writes and deleted participants must be resolved before an update. Pending,
unsubmitted restarts are superseded. Adding/removing files, changing targets or
changing the policy still requires a new beta.
## API and storage ## API and storage
`POST /api/betas` accepts `testPlan`. `POST /api/betas` accepts `testPlan`.
@@ -228,6 +254,12 @@ recent evidence under `test`.
`GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are `GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are
copied into a beta; later edits do not change an active plan. copied into a beta; later edits do not change an active plan.
`POST /api/betas/:id/revisions` accepts multipart `expectedRevision`, optional
`notes`, and replacement uploads named `file:<scriptId>`. A stale revision is
rejected. `GET /api/betas/:id` includes the current `revision` and `revisions`
history; `GET /api/betas/:id/revisions/:revision/files/:scriptId` downloads an
archived file (or the original pinned file before its first update).
Internal progress records support attempts and recovery cycles. The reporter Internal progress records support attempts and recovery cycles. The reporter
does not know about or supply these records. Legacy betas without a test plan does not know about or supply these records. Legacy betas without a test plan
also use `game_end` at round 2 or later for their match-completion gate. also use `game_end` at round 2 or later for their match-completion gate.
+11
View File
@@ -0,0 +1,11 @@
import type { BetaTestView } from './beta-plan.js';
export interface BetaRevisionFile {
script_id: number; name: string; kind: string; path: string;
version_id: number; version: number; release_version: string | null; sha256: string;
}
export interface BetaRevision {
revision: number; started_at: string; notes: string | null;
files: BetaRevisionFile[];
results: { state: string; test: BetaTestView | null; servers: Array<{ server: string; stage: string; error: string | null }> } | null;
}
+1
View File
@@ -347,6 +347,7 @@ export function db(): Database.Database {
migrateTable(handle, 'scripts', SCRIPTS_COLUMNS); migrateTable(handle, 'scripts', SCRIPTS_COLUMNS);
migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS); migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS);
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS); migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS); migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS); migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
migrateTable(handle, 'server_launch_collections', [ migrateTable(handle, 'server_launch_collections', [
+27
View File
@@ -180,6 +180,32 @@ CREATE TABLE IF NOT EXISTS betas (
CREATE INDEX IF NOT EXISTS idx_betas_state ON betas(state); CREATE INDEX IF NOT EXISTS idx_betas_state ON betas(state);
CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id); CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id);
-- Revisions keep the beta identity and original rollback images. Archived
-- files are independent copies so version cleanup cannot erase this history.
CREATE TABLE IF NOT EXISTS beta_revisions (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
revision INTEGER NOT NULL,
started_at TEXT NOT NULL,
notes TEXT,
results_json TEXT,
PRIMARY KEY (beta_id, revision)
);
CREATE TABLE IF NOT EXISTS beta_revision_files (
beta_id INTEGER NOT NULL,
revision INTEGER NOT NULL,
script_id INTEGER NOT NULL,
name TEXT NOT NULL,
kind TEXT NOT NULL,
path TEXT NOT NULL,
version_id INTEGER NOT NULL,
version INTEGER NOT NULL,
release_version TEXT,
sha256 TEXT NOT NULL,
content BLOB NOT NULL,
PRIMARY KEY (beta_id, revision, script_id),
FOREIGN KEY (beta_id, revision) REFERENCES beta_revisions(beta_id, revision) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS beta_servers ( CREATE TABLE IF NOT EXISTS beta_servers (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL, server TEXT NOT NULL,
@@ -208,6 +234,7 @@ CREATE TABLE IF NOT EXISTS beta_test_plans (
CREATE TABLE IF NOT EXISTS beta_test_runs ( CREATE TABLE IF NOT EXISTS beta_test_runs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
revision INTEGER NOT NULL DEFAULT 1,
server TEXT NOT NULL, server TEXT NOT NULL,
phase INTEGER NOT NULL DEFAULT 0, phase INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL, created_at TEXT NOT NULL,
+29
View File
@@ -0,0 +1,29 @@
import { db } from '../db/index.js';
import type { BetaRevision } from '../../beta-revision.js';
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
{ revision: number; started_at: string } | undefined
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
}
/** Called in the revision transaction, before replacing the pinned bundle. */
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
.run(id, revision, startedAt, notes);
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content)
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ?`).run(revision, id);
}
export function betaRevisionHistory(id: number): BetaRevision[] {
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
return rows.map(({ results_json, ...row }) => ({ ...row,
results: results_json ? JSON.parse(results_json) : null,
files: db().prepare(`SELECT script_id, name, kind, path, version_id, version, release_version, sha256
FROM beta_revision_files WHERE beta_id = ? AND revision = ? ORDER BY script_id`).all(id, row.revision) as BetaRevision['files']
}));
}
+4 -3
View File
@@ -6,6 +6,7 @@ import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan
import type { GameTelemetry } from '../../telemetry.js'; import type { GameTelemetry } from '../../telemetry.js';
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js'; import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { currentBetaRevision } from './beta-revisions.js';
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string } export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null } export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
@@ -33,8 +34,8 @@ export async function prepareRun(betaId: number, server: string, phase = 0): Pro
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta'); if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
db().transaction(() => { db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server); db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)') db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics())); .run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision);
})(); })();
return currentRun(betaId, server)!; return currentRun(betaId, server)!;
} }
@@ -97,5 +98,5 @@ export function testView(betaId: number): BetaTestView | null {
}), }),
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] }; WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] };
} }
+125 -3
View File
@@ -1,5 +1,6 @@
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
import { storedPlan } from './beta-tests.js'; import { storedPlan, testView, activity, setOutcome } from './beta-tests.js';
import { archiveBetaRevision, currentBetaRevision } from './beta-revisions.js';
import type { BetaPromotionPreview } from '../../beta-promotion.js'; import type { BetaPromotionPreview } from '../../beta-promotion.js';
const operations = new Set<number>(); const operations = new Set<number>();
@@ -28,17 +29,18 @@ import {
listVersions, listVersions,
serversMatchingScope, serversMatchingScope,
setScope, setScope,
uploadNewVersion,
type ScopeType, type ScopeType,
type ScriptRow, type ScriptRow,
type ScriptVersionRow type ScriptVersionRow
} from './scripts.js'; } from './scripts.js';
import { composeServerCfg } from './config-compose.js'; import { composeServerCfg } from './config-compose.js';
import { getServerScript, removeScript } from './inventory.js'; import { getServerScript, removeScript } from './inventory.js';
import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js'; import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
import { emit } from '../pipeline/bus.js'; import { emit } from '../pipeline/bus.js';
import { cancelProvisioningJob } from '../provisioning/cancellation.js'; import { cancelProvisioningJob } from '../provisioning/cancellation.js';
import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js'; import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js';
import { requestRestart } from '../provisioning/requests.js'; import { requestRestart, releaseRestartRequest } from '../provisioning/requests.js';
import type { BetaStage } from '../pipeline/bus.js'; import type { BetaStage } from '../pipeline/bus.js';
// Whether an artifact writes a dedicated on-disk file (script/plugin) or // Whether an artifact writes a dedicated on-disk file (script/plugin) or
@@ -497,6 +499,124 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
}; };
} }
export interface BetaRevisionUpload { scriptId: number; content: Buffer; releaseVersion?: string }
/** Replace existing bundle members; commit files, deadline and evidence reset together. */
export async function reviseBeta(betaId: number, expectedRevision: number, uploads: BetaRevisionUpload[], notes = ''): Promise<{ revision: number }> {
// Drain a runner already observing the previous revision before resetting it.
const { betaTick } = await import('../pipeline/beta-runner.js');
await betaTick();
return exclusive(betaId, async () => {
const beta = getBeta(betaId);
if (!beta) throw new BetaError('Beta not found', 'no-such-beta');
if (!['running', 'ready-promote'].includes(beta.state)) throw new BetaError('Beta is closed', 'invalid-state');
const current = currentBetaRevision(betaId);
if (current.revision !== expectedRevision) throw new BetaError('The beta changed. Reload and review the current revision.', 'preview-changed');
if (!uploads.length || new Set(uploads.map(file => file.scriptId)).size !== uploads.length) {
throw new BetaError('Choose at least one replacement, once per file.', 'invalid-state');
}
if (notes.length > 1000) throw new BetaError('Change notes must be at most 1000 characters.', 'invalid-state');
const bundle = listBetaScripts(betaId);
for (const file of uploads) {
const member = bundle.find(row => row.script_id === file.scriptId);
if (!member) throw new BetaError('Updates can only replace files already in this beta.', 'invalid-state');
if (!file.content.length) throw new BetaError('Choose non-empty replacement files.', 'invalid-state');
if (getVersion(member.script_version_id)!.sha256 === sha256Buffer(file.content)) {
throw new BetaError(`${getScript(file.scriptId)!.name}: the replacement is identical to the current version.`, 'invalid-state');
}
}
const servers = listBetaServers(betaId);
if (!servers.length || servers.some(server => !isActiveBetaServer(betaId, server.server))) {
throw new BetaError('An original beta server is missing. Roll back and open a new beta.', 'invalid-state');
}
ensureHostSchema(db());
assertNativeActivationSettled(betaId);
if (stagedFileChanges('beta', String(betaId)).some(row => row.state === 'applying')) {
throw new BetaError('A file write was interrupted. Resolve it before updating the beta.', 'conflict');
}
const jobs = db().prepare(`SELECT DISTINCT d.id, d.state, d.submitted_at, n.operation_id FROM server_deployments d
JOIN provisioning_sources s ON s.operation_id = d.id LEFT JOIN native_provisioning n ON n.deployment_id = d.id
WHERE s.kind = 'beta' AND s.source_id = ?`).all(String(betaId)) as
Array<{ id: string; state: string; submitted_at: string | null; operation_id: string | null }>;
if (jobs.some(job => ['running', 'awaiting-health'].includes(job.state)
|| (job.state === 'interrupted' && servers.some(server => server.provisioning_id === job.id))
|| (['waiting', 'queued'].includes(job.state) && (job.submitted_at || job.operation_id)))) {
throw new BetaError('Wait for the current activation and recovery to finish before updating this beta.', 'conflict');
}
const stored = storedPlan(betaId);
const plan = stored ? parseBetaPlan(JSON.parse(stored.plan)) : null;
const revision = current.revision + 1;
const at = nowIso();
db().transaction(() => {
archiveBetaRevision(betaId, current.revision, current.started_at, null);
db().prepare('UPDATE beta_revisions SET results_json = ? WHERE beta_id = ? AND revision = ?')
.run(JSON.stringify({ state: beta.state, test: testView(betaId), servers }), betaId, current.revision);
for (const upload of uploads) {
const version = uploadNewVersion(upload.scriptId, upload.content, notes.trim() || undefined, upload.releaseVersion);
db().prepare('UPDATE beta_scripts SET script_version_id = ? WHERE beta_id = ? AND script_id = ?')
.run(version.id, betaId, upload.scriptId);
db().prepare('UPDATE betas SET script_version_id = ? WHERE id = ? AND script_id = ?')
.run(version.id, betaId, upload.scriptId);
}
// Preflight all targets before cancelling work. Errors roll back the
// uploads and history as well as the beta changes.
const changes = new Map(servers.map(server => [server.server, revisionFileChanges(betaId, server.server)]));
// Betas created before staged activation may only have deployed_at
// and snapshots. Record their first writes before replacing staging.
for (const server of servers.filter(server => server.deployed_at)) {
for (const change of changes.get(server.server)!) {
if (isFileApplied('beta', String(betaId), server.server, change.kind, change.relPath) === null) {
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(betaId, server.server, change.relPath);
}
}
}
for (const job of jobs.filter(job => ['waiting', 'queued'].includes(job.state))) {
cancelProvisioningJob(job.id, `Superseded by beta #${betaId} revision ${revision}`);
}
for (const server of servers) releaseRestartRequest(server.server, { kind: 'beta', id: String(betaId) });
// New IDs prevent historical host receipts from resolving to a newer
// revision's desired bytes. Original before images stay intact.
db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId));
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(at, betaId);
db().prepare(`UPDATE beta_servers SET stage = 'queued', error = NULL, deployed_at = NULL,
restarted_at = NULL, healthy_at = NULL, match_seen_at = NULL, updated_at = ? WHERE beta_id = ?`).run(at, betaId);
db().prepare("UPDATE betas SET state = 'running' WHERE id = ?").run(betaId);
setOutcome(betaId, 'active', null);
archiveBetaRevision(betaId, revision, at, notes.trim() || null);
// The original first wave follows selection order; later waves are
// ordered by the runner. Preserve that canary selection on revision.
const firstWave = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ? ORDER BY rowid LIMIT ?')
.all(betaId, plan?.waveSize || servers.length) as { server: string }[];
for (const server of firstWave) {
for (const change of changes.get(server.server)!) queueFileChange(change);
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty' WHERE beta_id = ? AND server = ?").run(betaId, server.server);
}
activity(betaId, `Revision ${revision} submitted; fresh tests and testing window${notes.trim() ? `: ${notes.trim()}` : ''}`);
})();
emit({ type: 'beta-state', betaId, state: 'running' });
return { revision };
});
}
function revisionFileChanges(betaId: number, name: string): Parameters<typeof queueFileChange>[0][] {
const resolved = listBetaScripts(betaId).map(row => ({ script: getScript(row.script_id)!, version: getVersion(row.script_version_id)! }));
const server = getServer(name)!;
const changes: Parameters<typeof queueFileChange>[0][] = resolved.filter(row => !isCollapsingKind(row.script.kind))
.map(({ script, version }) => ({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: script.kind,
relPath: script.target_rel_path, content: version.content, scriptVersionId: version.id, inventorySource: `beta:${betaId}` }));
if (resolved.some(row => isCollapsingKind(row.script.kind))) {
const composed = composeCfgForServer(name, server.game, server.map, {
inBetaBaseline: resolved.find(row => row.script.kind === 'baseline')?.version,
inBetaOverlays: resolved.filter(row => row.script.kind === 'config')
});
if (!composed) throw new BetaError(`No baseline available for ${name}.`, 'invalid-state');
changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: 'config', relPath: configFileOf(name),
content: Buffer.from(composed.text), inventorySource: `beta:${betaId}` });
}
return changes;
}
/** Apply a reserved wave only when its predecessors have passed. */ /** Apply a reserved wave only when its predecessors have passed. */
export async function deployBetaServers(betaId: number, names: string[]): Promise<void> { export async function deployBetaServers(betaId: number, names: string[]): Promise<void> {
return exclusive(betaId, () => deployBetaServersInternal(betaId, names)); return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
@@ -620,6 +740,8 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = []; const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = [];
const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?'); const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?');
const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => { const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => {
// A queued revision may supersede an applied one before its own activation.
if (changed.get(betaId, bs.server, path)) return true;
const staged = isFileApplied('beta', String(betaId), bs.server, kind, path); const staged = isFileApplied('beta', String(betaId), bs.server, kind, path);
if (staged !== null) return staged; if (staged !== null) return staged;
// Existing betas predating staging wrote their files before deployed_at. // Existing betas predating staging wrote their files before deployed_at.
+16 -9
View File
@@ -80,6 +80,21 @@ function readExisting(path: string): Buffer | null {
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; }
} }
/** The first activation owns the rollback image, including an absent file. */
export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | null): void {
if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
.get(row.source_id, row.server, row.rel_path)) return;
const prior = getServerScript(row.server, row.rel_path);
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, row.server, row.rel_path, before, sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
db().prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(row.source_id, row.server, row.rel_path);
}
/** Only called after occupancy checks and reservation of the restart slot. /** Only called after occupancy checks and reservation of the restart slot.
* Synchronous file replacement keeps cancellation, new requests, and deletion * Synchronous file replacement keeps cancellation, new requests, and deletion
* from interleaving between the final gates and dispatching the restart. * from interleaving between the final gates and dispatching the restart.
@@ -111,17 +126,9 @@ export function applyStagedFilesForOperation(operationId: string, server: string
for (const { row, path } of plans) { for (const { row, path } of plans) {
// Earlier coalesced campaigns may have just updated the same path. // Earlier coalesced campaigns may have just updated the same path.
const before = readExisting(path); const before = readExisting(path);
const prior = getServerScript(server, row.rel_path);
db().transaction(() => { db().transaction(() => {
if (row.source_kind === 'beta') { if (row.source_kind === 'beta') {
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots captureBetaBeforeImage(row, before);
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
.run(row.source_id, server, row.rel_path, before, sha256Buffer(before), before.length);
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, server, row.rel_path, row.kind);
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
.run(row.source_id, server, row.rel_path);
} }
if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET
verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`) verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`)
+3 -3
View File
@@ -1,5 +1,6 @@
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js'; import { currentBetaRevision } from '../fleet/beta-revisions.js';
import { betaOperationInFlight, deployBetaServers, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js'; import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js'; import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
@@ -64,7 +65,6 @@ async function tick(): Promise<void> {
async function advancePlan(stored: StoredPlan): Promise<void> { async function advancePlan(stored: StoredPlan): Promise<void> {
const id = stored.beta_id; const id = stored.beta_id;
const plan = parseBetaPlan(JSON.parse(stored.plan)); const plan = parseBetaPlan(JSON.parse(stored.plan));
const beta = getBeta(id)!;
const servers = listBetaServers(id); const servers = listBetaServers(id);
// Preserve the original coverage and evidence. A replacement with the same // Preserve the original coverage and evidence. A replacement with the same
// name cannot complete a test that began on a different server. // name cannot complete a test that began on a different server.
@@ -74,7 +74,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`); setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
return; return;
} }
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) { if (stored.outcome !== 'passed' && Date.now() - Date.parse(currentBetaRevision(id).started_at) >= plan.timeoutHours * 3_600_000) {
betaState(id, false); betaState(id, false);
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence'); setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return; return;
+3 -8
View File
@@ -22,9 +22,9 @@ import { fleetSettings } from '../settings.js';
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js'; import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js'; import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js'; import { recordScript, removeScript } from '../fleet/inventory.js';
import { sha256Buffer } from '../fleet/files.js'; import { sha256Buffer } from '../fleet/files.js';
import type { StagedFileChange } from '../fleet/staged-files.js'; import { captureBetaBeforeImage, type StagedFileChange } from '../fleet/staged-files.js';
import type { LaunchPreview } from '../../deployment-plan.js'; import type { LaunchPreview } from '../../deployment-plan.js';
import type { ProvisioningAction } from '../../provisioning.js'; import type { ProvisioningAction } from '../../provisioning.js';
import { hostAgentUpdatesEnabled } from '../../hosts.js'; import { hostAgentUpdatesEnabled } from '../../hosts.js';
@@ -227,13 +227,8 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange; const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
if (!row || row.state === 'applied') continue; if (!row || row.state === 'applied') continue;
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
const prior = getServerScript(job.server, row.rel_path);
if (row.source_kind === 'beta') { if (row.source_kind === 'beta') {
if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`) captureBetaBeforeImage(row, bytes);
.run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length);
db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=?
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind);
db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path);
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
} }
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?') if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
@@ -0,0 +1,100 @@
<script lang="ts">
import Dialog from './primitives/Dialog.svelte';
import Button from './primitives/Button.svelte';
import DiffViewer from './DiffViewer.svelte';
import ServerIdentity from './ServerIdentity.svelte';
import { artifactVersionLabel } from '$lib/artifact-version.js';
interface FileEntry { id: number; name: string; kind: string; target_rel_path: string;
version: { version: number; release_version: string | null; sha256: string } | null }
let { open = $bindable(false), betaId, revision, files, servers, timeoutHours, onUpdate }:
{ open?: boolean; betaId: number; revision: number; files: FileEntry[]; servers: string[];
timeoutHours?: number; onUpdate: () => Promise<void> } = $props();
interface Replacement { file: File; sha: string; before: string | null; after: string | null }
let replacements = $state<Record<number, Replacement>>({});
let notes = $state(''), error = $state(''), reviewing = $state(false), busy = $state(false);
const count = $derived(Object.keys(replacements).length);
$effect(() => { if (open) { replacements = {}; notes = ''; error = ''; reviewing = false; } });
function fileUrl(scriptId: number) { return `/api/betas/${betaId}/revisions/${revision}/files/${scriptId}`; }
async function choose(entry: FileEntry, file?: File) {
delete replacements[entry.id]; error = '';
if (!file) return;
busy = true;
try {
if (!file.size) throw new Error('Choose a non-empty file.');
const bytes = await file.arrayBuffer();
const inspection = new FormData(); inspection.set('content', file);
const inspected = await fetch('/api/scripts/inspect', { method: 'POST', body: inspection });
const inspectionBody = await inspected.json();
if (!inspected.ok) throw new Error(inspectionBody.error ?? 'Could not inspect replacement file.');
const sha = inspectionBody.sha256 as string;
if (sha === entry.version?.sha256) throw new Error(`${entry.name}: this file is identical to the current version.`);
let before: string | null = null, after: string | null = null;
if (entry.kind !== 'plugin' && bytes.byteLength <= 128 * 1024) {
const response = await fetch(fileUrl(entry.id));
if (!response.ok) throw new Error('Could not load the current file for review. Reload the beta and try again.');
const old = await response.arrayBuffer();
if (old.byteLength <= 128 * 1024 && !new Uint8Array(old).includes(0) && !new Uint8Array(bytes).includes(0)) {
before = new TextDecoder().decode(old); after = new TextDecoder().decode(bytes);
}
}
replacements[entry.id] = { file, sha, before, after };
} catch (err) { error = (err as Error).message; }
finally { busy = false; }
}
async function deploy() {
busy = true; error = '';
try {
const form = new FormData();
form.set('expectedRevision', String(revision)); form.set('notes', notes);
for (const [scriptId, replacement] of Object.entries(replacements)) form.set(`file:${scriptId}`, replacement.file);
const response = await fetch(`/api/betas/${betaId}/revisions`, { method: 'POST', body: form });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not update beta.');
open = false; await onUpdate();
} catch (err) { error = (err as Error).message; }
finally { busy = false; }
}
</script>
<Dialog bind:open title={reviewing ? `Review revision ${revision + 1}` : 'Update beta'}
description={`Replace files in beta #${betaId} and repeat its tests on the same servers.`} {busy} wide>
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
{#if !reviewing}
<p>Choose replacements for the files you changed.</p>
<div class="files">{#each files as entry}
<label><strong>{entry.name} · {artifactVersionLabel(entry.version)}</strong><code>{entry.target_rel_path}</code>
<input type="file" disabled={busy} onchange={event => choose(entry, event.currentTarget.files?.[0])} />
{#if replacements[entry.id]}<span>Replacement ready for review</span>{/if}
</label>
{/each}</div>
<label class="notes">Change note (optional)<textarea bind:value={notes} maxlength="1000" rows="3" disabled={busy}></textarea></label>
{:else}
<p><strong>{count} file{count === 1 ? '' : 's'} changed.</strong> All {servers.length} selected servers will repeat the test policy, using its existing wave order. Each server installs the bundle and restarts when empty.</p>
<p>Previous results move to revision history. Promotion requires this revision to pass.
{#if timeoutHours}A fresh {timeoutHours}-hour test window starts when you deploy, including time waiting for empty servers.{/if}
Rejecting the beta still restores the files from before revision 1.</p>
<details open><summary>Servers ({servers.length})</summary><ul>{#each servers as name}<li><ServerIdentity {name}/></li>{/each}</ul></details>
{#each files.filter(entry => replacements[entry.id]) as entry}
{@const replacement = replacements[entry.id]}
<section><h3>{entry.name}</h3><p><code>{entry.target_rel_path}</code> ← {replacement.file.name} ({replacement.file.size.toLocaleString()} bytes)</p>
<p class="hash">SHA {entry.version?.sha256.slice(0,12)} → {replacement.sha.slice(0,12)}</p>
{#if replacement.after !== null}<DiffViewer before={replacement.before} after={replacement.after} verb="updated" relPath={entry.target_rel_path}/>
{:else}<p>Binary or large file: review the uploaded file and checksum.</p>{/if}
</section>
{/each}
{#if notes}<p>Change note: {notes}</p>{/if}
{/if}
{#snippet actions()}
<Button onclick={() => open = false} disabled={busy}>Cancel</Button>
{#if reviewing}<Button onclick={() => reviewing = false} disabled={busy}>Back</Button><Button variant="primary" onclick={deploy} disabled={busy} loading={busy}>Deploy revision {revision + 1}</Button>
{:else}<Button variant="primary" onclick={() => reviewing = true} disabled={busy || !count}>Review update</Button>{/if}
{/snippet}
</Dialog>
<style>
.files, label { display:grid; gap:12px; } .files { margin:20px 0; }
.files label { padding:16px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); }
p { line-height:1.6; } code, .hash { overflow-wrap:anywhere; } .hash, label span { color:var(--text-secondary); font-size:13px; }
li { margin:8px 0; } textarea { width:100%; } section { margin-top:24px; }
</style>
@@ -50,7 +50,7 @@
{#if ['paused','failed'].includes(test.outcome) && !expired}<Button size="sm" disabled={busy || !!retryBlocked} onclick={()=>action('retry')}>Start fresh attempt</Button>{/if} {#if ['paused','failed'].includes(test.outcome) && !expired}<Button size="sm" disabled={busy || !!retryBlocked} onclick={()=>action('retry')}>Start fresh attempt</Button>{/if}
</div>{/if} </div>{/if}
{#if retryBlocked && ['paused','failed'].includes(test.outcome)}<p>{retryBlocked}</p>{/if} {#if retryBlocked && ['paused','failed'].includes(test.outcome)}<p>{retryBlocked}</p>{/if}
{#if expired}<p class="guidance">The testing window ended. Roll back this beta first, then choose “Reuse files and test plan” to open a new beta with a longer window.</p> {#if expired}<p class="guidance">The testing window ended. Use “Update beta” to test changed files in a fresh revision, or roll back and reuse the plan with a longer window.</p>
{:else if test.outcome === 'paused' && !hasDeletedServers}<p class="guidance">Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.</p>{/if} {:else if test.outcome === 'paused' && !hasDeletedServers}<p class="guidance">Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.</p>{/if}
{/if} {/if}
{#if error}<p role="alert" class="failed">{error}</p>{/if} {#if error}<p role="alert" class="failed">{error}</p>{/if}
+2 -1
View File
@@ -10,6 +10,7 @@ import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
import { kick } from '$lib/server/pipeline/worker.js'; import { kick } from '$lib/server/pipeline/worker.js';
import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js'; import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js';
import { storedPlan } from '$lib/server/fleet/beta-tests.js'; import { storedPlan } from '$lib/server/fleet/beta-tests.js';
import { currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
export async function GET({ url }: { url: URL }) { export async function GET({ url }: { url: URL }) {
const stateFilter = url.searchParams.get('state') as const stateFilter = url.searchParams.get('state') as
@@ -38,7 +39,7 @@ export async function GET({ url }: { url: URL }) {
...b, ...b,
testOutcome: test?.outcome ?? null, testOutcome: test?.outcome ?? null,
testReason: test?.reason ?? null, testReason: test?.reason ?? null,
deadlineAt: plan ? new Date(Date.parse(b.opened_at) + plan.timeoutHours * 3600000).toISOString() : null, deadlineAt: plan ? new Date(Date.parse(currentBetaRevision(b.id).started_at) + plan.timeoutHours * 3600000).toISOString() : null,
scripts, scripts,
servers: listBetaServers(b.id) servers: listBetaServers(b.id)
}; };
+3 -1
View File
@@ -3,6 +3,7 @@ import { db } from '$lib/server/db/index.js';
import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js'; import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js';
import { getScript, getVersion } from '$lib/server/fleet/scripts.js'; import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
import { testView } from '$lib/server/fleet/beta-tests.js'; import { testView } from '$lib/server/fleet/beta-tests.js';
import { betaRevisionHistory, currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
function parseId(raw: string): number | null { function parseId(raw: string): number | null {
const n = Number.parseInt(raw, 10); const n = Number.parseInt(raw, 10);
@@ -37,5 +38,6 @@ export async function GET({ params }: { params: { id: string } }) {
failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null; failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1") const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1")
.get(id) as { id: string } | undefined; .get(id) as { id: string } | undefined;
return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null }); return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null,
revision: currentBetaRevision(id), revisions: betaRevisionHistory(id) });
} }
@@ -0,0 +1,28 @@
import { json } from '@sveltejs/kit';
import { BetaError, reviseBeta, type BetaRevisionUpload } from '$lib/server/fleet/beta.js';
import { ScriptError } from '$lib/server/fleet/scripts.js';
import { kick } from '$lib/server/pipeline/worker.js';
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
const id = Number(params.id);
if (!Number.isSafeInteger(id) || id < 1) return json({ ok: false, error: 'Invalid beta ID' }, { status: 400 });
try {
const form = await request.formData();
const expectedRevision = Number(form.get('expectedRevision'));
if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 1) throw new Error('A reviewed revision is required.');
const uploads: BetaRevisionUpload[] = [];
for (const [key, value] of form) {
if (!key.startsWith('file:')) continue;
const scriptId = Number(key.slice(5));
if (!Number.isSafeInteger(scriptId) || scriptId < 1 || !(value instanceof File)) throw new Error('Invalid replacement file.');
uploads.push({ scriptId, content: Buffer.from(await value.arrayBuffer()),
releaseVersion: form.has(`release:${scriptId}`) ? String(form.get(`release:${scriptId}`)) : undefined });
}
const result = await reviseBeta(id, expectedRevision, uploads, String(form.get('notes') ?? ''));
void kick();
return json({ ok: true, result });
} catch (error) {
return json({ ok: false, error: (error as Error).message },
{ status: error instanceof BetaError || error instanceof ScriptError ? 409 : 400 });
}
}
@@ -0,0 +1,16 @@
import { db } from '$lib/server/db/index.js';
/** Current files can be reviewed before the first revision is archived. */
export async function GET({ params }: { params: { id: string; revision: string; scriptId: string } }) {
const values = [params.id, params.revision, params.scriptId].map(Number);
if (values.some(value => !Number.isSafeInteger(value) || value < 1)) return new Response('Invalid file', { status: 400 });
const [id, revision, scriptId] = values;
const file = db().prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?')
.get(id, revision, scriptId) as { content: Buffer } | undefined
?? (revision === 1 ? db().prepare(`SELECT v.content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id
WHERE bs.beta_id = ? AND bs.script_id = ? AND NOT EXISTS (SELECT 1 FROM beta_revisions WHERE beta_id = ?)`)
.get(id, scriptId, id) as { content: Buffer } | undefined : undefined);
if (!file) return new Response('File not found', { status: 404 });
return new Response(new Uint8Array(file.content), { headers: { 'Content-Type': 'application/octet-stream',
'Content-Disposition': 'attachment', 'Cache-Control': 'no-store' } });
}
+2 -3
View File
@@ -50,9 +50,8 @@ export async function POST({ params, request }: { params: { id: string }; reques
} }
setOutcome(id, 'active', null); setOutcome(id, 'active', null);
} else if (body.action === 'retry') { } else if (body.action === 'retry') {
// Fresh attempts keep their predecessors as audit history. Preserve the original deadline: // Fresh attempts keep history and the current revision's deadline.
// operators open another beta if the overall testing window has expired. if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Update the files to start a new revision, or roll back and open a new beta with a longer deadline');
if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) { if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta'); throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
} }
+26 -6
View File
@@ -11,6 +11,8 @@
import type { ActivityEvent } from '$lib/operation-flow.js'; import type { ActivityEvent } from '$lib/operation-flow.js';
import { pollVisible } from '$ui/poll.js'; import { pollVisible } from '$ui/poll.js';
import { confirmAction } from '$ui/confirm.js'; import { confirmAction } from '$ui/confirm.js';
import type { BetaRevision } from '$lib/beta-revision.js';
import BetaRevisionDialog from '$ui/components/BetaRevisionDialog.svelte';
import BetaTestProgress from '$ui/components/BetaTestProgress.svelte'; import BetaTestProgress from '$ui/components/BetaTestProgress.svelte';
import ServerIdentity from '$ui/components/ServerIdentity.svelte'; import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
@@ -38,10 +40,13 @@
let acting = $state<string|null>(null), request = 0; let acting = $state<string|null>(null), request = 0;
let reviewOpen = $state(false), preview = $state<BetaPromotionPreview|null>(null), reviewError = $state(''); let reviewOpen = $state(false), preview = $state<BetaPromotionPreview|null>(null), reviewError = $state('');
let now = $state(Date.now()); let now = $state(Date.now());
let revision = $state({ revision: 1, started_at: '' }), revisions = $state<BetaRevision[]>([]);
let updateOpen = $state(false), updateRevision = $state(1), updateFiles = $state<BetaFile[]>([]);
function openUpdate() { updateRevision = revision.revision; updateFiles = [...scripts]; updateOpen = true; }
const id = $derived(Number($page.params.id)); const id = $derived(Number($page.params.id));
const closed = $derived(beta?.state === 'promoted' || beta?.state === 'rolled-back'); const closed = $derived(beta?.state === 'promoted' || beta?.state === 'rolled-back');
const hasDeletedServers = $derived(servers.some(server => server.server_deleted_at)); const hasDeletedServers = $derived(servers.some(server => server.server_deleted_at));
const deadline = $derived(beta && test ? new Date(Date.parse(beta.opened_at) + test.plan.timeoutHours * 3600000) : null); const deadline = $derived(beta && test ? new Date(Date.parse(revision.started_at || beta.opened_at) + test.plan.timeoutHours * 3600000) : null);
const remaining = $derived(deadline ? Math.max(0, Math.ceil((deadline.getTime() - now) / 60000)) : 0); const remaining = $derived(deadline ? Math.max(0, Math.ceil((deadline.getTime() - now) / 60000)) : 0);
const deadlineText = $derived(remaining > 0 ? `${Math.floor(remaining / 60)}h ${remaining % 60}m remaining` : 'Testing window ended'); const deadlineText = $derived(remaining > 0 ? `${Math.floor(remaining / 60)}h ${remaining % 60}m remaining` : 'Testing window ended');
const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length); const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length);
@@ -49,7 +54,7 @@
const waiting = $derived(ranked.filter(server => server.stage !== 'ready')); const waiting = $derived(ranked.filter(server => server.stage !== 'ready'));
const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.' const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.'
: hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.' : hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.'
: test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.' : test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Update the files to start a new revision, or roll back and reuse this plan.'
: test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.' : test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.'
: test?.outcome === 'failed' ? 'Investigate the failed check before a fresh attempt, or roll back.' : test?.outcome === 'failed' ? 'Investigate the failed check before a fresh attempt, or roll back.'
: beta?.state === 'ready-promote' ? 'All required coverage passed. Review the full promotion impact before deploying to additional servers.' : beta?.state === 'ready-promote' ? 'All required coverage passed. Review the full promotion impact before deploying to additional servers.'
@@ -64,11 +69,12 @@
if (current !== request || requestedId !== id) return; if (current !== request || requestedId !== id) return;
test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null; test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null;
rollbackCampaign = body.rollback ?? null; rollbackCampaign = body.rollback ?? null;
revision = body.revision; revisions = body.revisions ?? [];
now = Date.now(); loadError = ''; now = Date.now(); loadError = '';
} catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; } } catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; }
finally { if (current === request && requestedId === id) loading = false; } finally { if (current === request && requestedId === id) loading = false; }
} }
$effect(() => { id; beta = null; loading = true; loadError = ''; flash = ''; success = ''; reviewOpen = false; void load(); }); $effect(() => { id; beta = null; loading = true; loadError = ''; flash = ''; success = ''; reviewOpen = false; updateOpen = false; void load(); });
onMount(() => pollVisible(load, 5000, false)); onMount(() => pollVisible(load, 5000, false));
async function reviewPromotion() { async function reviewPromotion() {
reviewOpen = true; preview = null; reviewError = ''; acting = 'preview'; reviewOpen = true; preview = null; reviewError = ''; acting = 'preview';
@@ -129,7 +135,7 @@
{:else if !beta && !loadError}<EmptyState title="Beta not found" description="Return to Betas to find another test." /> {:else if !beta && !loadError}<EmptyState title="Beta not found" description="Return to Betas to find another test." />
{:else if beta} {:else if beta}
<div class="page"> <div class="page">
<SectionHeader title={beta.notes || `Beta #${beta.id}`} subtitle={beta.notes ? `Beta #${beta.id}` : undefined}> <SectionHeader title={beta.notes || `Beta #${beta.id}`} subtitle={`Beta #${beta.id} · Revision ${revision.revision}`}>
{#snippet crumbs()}<Breadcrumbs items={[{label:'Betas',href:'/betas'},{label:`#${id}`}]} />{/snippet} {#snippet crumbs()}<Breadcrumbs items={[{label:'Betas',href:'/betas'},{label:`#${id}`}]} />{/snippet}
{#snippet actions()}<StatusBadge state={test && !closed ? test.outcome : beta?.state ?? 'running'} />{/snippet} {#snippet actions()}<StatusBadge state={test && !closed ? test.outcome : beta?.state ?? 'running'} />{/snippet}
</SectionHeader> </SectionHeader>
@@ -139,12 +145,13 @@
<p>{nextAction}</p> <p>{nextAction}</p>
{#if test?.reason}<p class="reason">{test.reason}</p>{/if} {#if test?.reason}<p class="reason">{test.reason}</p>{/if}
<div class="timing"><span>Started <time datetime={beta.opened_at}>{$timeFormat.dateTime(beta.opened_at)}</time></span> <div class="timing"><span>Started <time datetime={beta.opened_at}>{$timeFormat.dateTime(beta.opened_at)}</time></span>
{#if revision.revision > 1}<span>Revision {revision.revision} started {$timeFormat.dateTime(revision.started_at)}</span>{/if}
{#if deadline && !closed}<strong>Ends <time datetime={deadline.toISOString()}>{$timeFormat.dateTime(deadline)}</time> · {deadlineText}</strong>{/if} {#if deadline && !closed}<strong>Ends <time datetime={deadline.toISOString()}>{$timeFormat.dateTime(deadline)}</time> · {deadlineText}</strong>{/if}
{#if beta.closed_at}<span>Finished {$timeFormat.dateTime(beta.closed_at)}</span>{/if} {#if beta.closed_at}<span>Finished {$timeFormat.dateTime(beta.closed_at)}</span>{/if}
</div> </div>
{#if !closed && test}<p class="muted">Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline.</p>{/if} {#if !closed && test}<p class="muted">Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline. Deploying a revision starts a fresh window.</p>{/if}
<div class="actions"> <div class="actions">
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if} {#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button onclick={openUpdate} disabled={hasDeletedServers || acting !== null}>Update beta</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
{#if closed}<Button href={contextHref(`/betas/new?reuse=${id}`, $page.url)}>Reuse files and test plan</Button>{/if} {#if closed}<Button href={contextHref(`/betas/new?reuse=${id}`, $page.url)}>Reuse files and test plan</Button>{/if}
{#if promotion}<Button href={contextHref(`/campaigns/${encodeURIComponent(promotion.id)}`, $page.url)}>View campaign</Button>{/if} {#if promotion}<Button href={contextHref(`/campaigns/${encodeURIComponent(promotion.id)}`, $page.url)}>View campaign</Button>{/if}
{#if rollbackCampaign}<Button href={contextHref(`/campaigns/${encodeURIComponent(rollbackCampaign.id)}`, $page.url)}>View rollback campaign</Button>{/if} {#if rollbackCampaign}<Button href={contextHref(`/campaigns/${encodeURIComponent(rollbackCampaign.id)}`, $page.url)}>View rollback campaign</Button>{/if}
@@ -163,10 +170,22 @@
<details class="ui-panel"><summary>Deployment progress ({servers.length} servers)</summary><div class="server-list"> <details class="ui-panel"><summary>Deployment progress ({servers.length} servers)</summary><div class="server-list">
{#each ranked as server}<div class="server-row"><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} href={contextHref(`/servers/${encodeURIComponent(server.server)}`, $page.url)}/><StatusBadge state={server.stage}/><p>Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}</p>{#if server.provisioning_id}<a href={contextHref(`/provisioning/${encodeURIComponent(server.provisioning_id)}`, $page.url)}>View provisioning</a>{/if}{#if server.error}<p class="ui-error">{server.error}</p>{/if}{#if server.stage === 'failed' && !server.server_deleted_at && !test && !closed}<Button size="sm" onclick={() => retry(server.server)} disabled={acting !== null} loading={acting === `retry:${server.server}`}>Retry server</Button>{/if}</div>{/each} {#each ranked as server}<div class="server-row"><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} href={contextHref(`/servers/${encodeURIComponent(server.server)}`, $page.url)}/><StatusBadge state={server.stage}/><p>Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}</p>{#if server.provisioning_id}<a href={contextHref(`/provisioning/${encodeURIComponent(server.provisioning_id)}`, $page.url)}>View provisioning</a>{/if}{#if server.error}<p class="ui-error">{server.error}</p>{/if}{#if server.stage === 'failed' && !server.server_deleted_at && !test && !closed}<Button size="sm" onclick={() => retry(server.server)} disabled={acting !== null} loading={acting === `retry:${server.server}`}>Retry server</Button>{/if}</div>{/each}
</div></details> </div></details>
{#if revisions.length}
<details class="ui-panel"><summary>Revision history ({revisions.length})</summary><div class="revision-list">
{#each revisions as item}<details><summary>Revision {item.revision} · {item.revision === revision.revision ? 'Current' : item.results?.test?.outcome ?? item.results?.state ?? 'Superseded'} · {$timeFormat.dateTime(item.started_at)}</summary>
{#if item.notes}<p>{item.notes}</p>{/if}
<div class="file-list">{#each item.files as file}<div><a href={`/api/betas/${id}/revisions/${item.revision}/files/${file.script_id}`} download={file.path.split('/').pop()}>{file.name} · {artifactVersionLabel(file)}</a><code>{file.path}</code><span>{file.sha256.slice(0,12)}</span></div>{/each}</div>
{#if item.results?.test}<BetaTestProgress test={item.results.test} betaId={id} closed={true} onUpdate={load} />
{:else if item.results}<ul>{#each item.results.servers as server}<li>{server.server} · {server.stage}{server.error ? ` · ${server.error}` : ''}</li>{/each}</ul>{/if}
</details>{/each}
</div></details>
{/if}
<OperationHistory {events} active={!closed} showOverview={false}/> <OperationHistory {events} active={!closed} showOverview={false}/>
</div> </div>
{/if} {/if}
<BetaRevisionDialog bind:open={updateOpen} betaId={id} revision={updateRevision} files={updateFiles} servers={servers.map(server => server.server)} timeoutHours={test?.plan.timeoutHours} onUpdate={load} />
<Dialog bind:open={reviewOpen} title="Review promotion" description="Review the exact files and server eligibility before installing the promoted versions." busy={acting === 'promote'} wide> <Dialog bind:open={reviewOpen} title="Review promotion" description="Review the exact files and server eligibility before installing the promoted versions." busy={acting === 'promote'} wide>
{#if acting === 'preview'}<p role="status">Preparing promotion impact…</p>{/if} {#if acting === 'preview'}<p role="status">Preparing promotion impact…</p>{/if}
{#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if} {#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if}
@@ -182,6 +201,7 @@
</Dialog> </Dialog>
<style> <style>
.revision-list { display:grid; gap:20px; margin-top:16px; }
.page { display:grid; gap:24px; } .page { display:grid; gap:24px; }
h2 { margin:0; font-size:18px; } p { line-height:1.6; overflow-wrap:anywhere; } h2 { margin:0; font-size:18px; } p { line-height:1.6; overflow-wrap:anywhere; }
.summary-heading,.actions,.timing { display:flex; align-items:center; gap:12px; flex-wrap:wrap; } .summary-heading,.actions,.timing { display:flex; align-items:center; gap:12px; flex-wrap:wrap; }
+24
View File
@@ -68,6 +68,30 @@ async function tick(seconds = 15, players = 4, healthy = true) {
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc'); const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
describe('durable beta test plans', () => { describe('durable beta test plans', () => {
it('requires fresh gameplay after a revision and promotes exactly the replacement version', async () => {
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
await tick(); await tick();
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
const { getBeta, listBetaScripts, reviseBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
const { getScript } = await import('../../src/lib/server/fleet/scripts.js');
const file = listBetaScripts(id)[0];
expect(getBeta(id)?.state).toBe('ready-promote');
await reviseBeta(id, 1, [{ scriptId: file.script_id, content: Buffer.from('main() { /* fixed */ }') }]);
const replacement = listBetaScripts(id)[0].script_version_id;
expect(replacement).not.toBe(file.script_version_id);
await postTelemetry(gameEvent('game_end')); // Still running the old revision.
expect(testView(id)?.evidence).toHaveLength(0);
await postTelemetry(gameEvent('heartbeat', 1, 0));
await tick(); await tick();
expect(getBeta(id)?.state).toBe('running');
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
expect(getBeta(id)?.state).toBe('ready-promote');
await promoteBeta(id);
expect(getScript(file.script_id)?.current_version_id).toBe(replacement);
});
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => { it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
const id = await openPlan(defaultPlan()); const id = await openPlan(defaultPlan());
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js'); const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
+222
View File
@@ -0,0 +1,222 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { activateFiles } from '../helpers/staged-files.js';
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
let fixture: Fixture;
const names = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture(names);
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
async function setup(options: { original?: string; waveSize?: number; kind?: 'script' | 'baseline'; legacy?: boolean } = {}) {
const scripts = await import('../../src/lib/server/fleet/scripts.js');
const beta = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
const kind = options.kind ?? 'script';
const path = join(destOfKind(names[0], kind), kind === 'script' ? 'revision.gsc' : configFileOf(names[0]));
if (options.original !== undefined) {
await mkdir(dirname(path), { recursive: true }); await writeFile(path, options.original);
}
const { script, version } = scripts.createScript({ name: 'Revision test', targetRelPath: kind === 'script' ? 'revision.gsc' : undefined,
kind, scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'script' ? 'revision one' : 'set sv_voice "1"\n') });
const plan = { ...defaultPlan(), waveSize: options.waveSize ?? 0, requirements: [requirement('manual', 'review')] };
const { betaId } = await beta.openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: options.waveSize ? names : [names[0]], waitForHealth: true, waitForMatch: false,
...(options.legacy ? {} : { testPlan: plan }) });
// Pause while arranging test state; revision submission explicitly starts testing.
if (!options.legacy) db().prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
return { ...scripts, ...beta, db: db(), script, version, betaId, path, plan };
}
describe('beta revisions', () => {
it('keeps the beta identity and archives files, sign-offs and evidence while resetting tests and the deadline', async () => {
const x = await setup();
await activateFiles('beta', x.betaId);
const { prepareRun, saveMetrics, readMetrics, evidence, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
const { currentBetaRevision, betaRevisionHistory } = await import('../../src/lib/server/fleet/beta-revisions.js');
const run = await prepareRun(x.betaId, names[0]);
const metrics = readMetrics(run); metrics.manual.review = 'Josh: verified'; saveMetrics(run, metrics);
evidence(run, 'old-signoff', 'manual-sign-off', { note: 'Josh: verified' });
x.db.prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(x.betaId);
x.db.prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(x.betaId);
const opened = x.getBeta(x.betaId)!.opened_at;
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.parse(opened) + 60_000);
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('revision two') }], 'Small fix'))
.resolves.toEqual({ revision: 2 });
expect(x.getBeta(x.betaId)).toMatchObject({ state: 'running', opened_at: opened });
expect(currentBetaRevision(x.betaId)).toEqual({ revision: 2, started_at: new Date().toISOString() });
expect(testView(x.betaId)).toMatchObject({ outcome: 'active', servers: [], evidence: [] });
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', restarted_at: null, deployed_at: null, provisioning_id: null });
const history = betaRevisionHistory(x.betaId);
expect(history[0]).toMatchObject({ revision: 2, notes: 'Small fix', results: null });
expect(history[1].results?.test?.servers[0].passed).toBe(true);
expect(history[1].results?.test?.evidence).toHaveLength(1);
expect(x.db.prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = 1').get(x.betaId))
.toEqual({ content: Buffer.from('revision one') });
expect(await readFile(x.path, 'utf8')).toBe('revision one');
await expect(x.promoteBeta(x.betaId)).rejects.toThrow('has not passed');
const nextRun = x.db.prepare('SELECT revision, metrics FROM beta_test_runs WHERE retired_at IS NULL').get() as { revision: number; metrics: string };
expect(nextRun.revision).toBe(2); expect(JSON.parse(nextRun.metrics).manual).toEqual({});
});
it.each(['original bytes', undefined])('restores the pre-beta state after multiple revisions (original=%s)', async original => {
const x = await setup({ original });
await activateFiles('beta', x.betaId);
for (let revision = 2; revision <= 3; revision++) {
await x.reviseBeta(x.betaId, revision - 1, [{ scriptId: x.script.id, content: Buffer.from(`revision ${revision}`) }]);
await activateFiles('beta', x.betaId);
}
expect(await readFile(x.path, 'utf8')).toBe('revision 3');
expect(x.db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get())
.toEqual({ had_original: original === undefined ? 0 : 1, prior_version_id: null });
const result = await x.rollbackBeta(x.betaId);
await activateFiles('campaign', result.deploymentId!);
if (original === undefined) expect(existsSync(x.path)).toBe(false);
else expect(await readFile(x.path, 'utf8')).toBe(original);
});
it.each([false, true])('rolls back a prior activation while its replacement is still queued (waves=%s)', async waves => {
const x = await setup({ original: 'before beta', waveSize: waves ? 1 : 0 });
await activateFiles('beta', x.betaId);
if (waves) {
await x.deployBetaServers(x.betaId, [names[1]]); await activateFiles('beta', x.betaId);
}
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('pending replacement') }]);
if (waves) expect(x.listBetaServers(x.betaId).map(row => row.stage)).toEqual(['awaiting-empty', 'queued']);
const result = await x.rollbackBeta(x.betaId);
expect(result.restored).toContain(names[0]);
if (waves) expect(result.dropped).toContain(names[1]);
await activateFiles('campaign', result.deploymentId!);
expect(await readFile(x.path, 'utf8')).toBe('before beta');
});
it('does not restore or delete files when no revision was ever activated', async () => {
const x = await setup({ original: 'untouched' });
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('new') }]);
expect(await x.rollbackBeta(x.betaId)).toMatchObject({ deploymentId: null });
expect(await readFile(x.path, 'utf8')).toBe('untouched');
});
it('preserves the original tracked inventory version across revisions', async () => {
const x = await setup({ original: 'tracked original' });
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
const prior = x.uploadNewVersion(x.script.id, Buffer.from('tracked original'));
recordScript(names[0], 'revision.gsc', prior.id, prior.sha256, prior.size, 'test');
await activateFiles('beta', x.betaId);
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
await activateFiles('beta', x.betaId);
expect(x.db.prepare('SELECT prior_version_id FROM beta_server_scripts').get()).toEqual({ prior_version_id: prior.id });
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
expect(getServerScript(names[0], 'revision.gsc')?.script_version_id).toBe(prior.id);
});
it('starts a revision with the same original canary selection', async () => {
const x = await setup({ waveSize: 1 });
await x.rollbackBeta(x.betaId);
const { betaId } = await x.openBeta({ scripts: [{ scriptId: x.script.id, scriptVersionId: x.version.id }],
servers: [...names].reverse(), waitForHealth: true, waitForMatch: false, testPlan: x.plan });
x.db.prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
await x.reviseBeta(betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
expect(x.listBetaServers(betaId).map(row => [row.server, row.stage]))
.toEqual([[names[0], 'queued'], [names[1], 'awaiting-empty']]);
});
it('allows updates after an interrupted attempt was superseded', async () => {
const x = await setup();
const { requestRestart, releaseRestartRequest } = await import('../../src/lib/server/provisioning/requests.js');
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
x.db.prepare("UPDATE server_deployments SET state = 'interrupted' WHERE id = ?").run(job.id);
releaseRestartRequest(names[0], { kind: 'beta', id: String(x.betaId) });
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]))
.resolves.toEqual({ revision: 2 });
});
it('recomposes config revisions and preserves the original shared file', async () => {
const x = await setup({ original: 'original config', kind: 'baseline' });
await activateFiles('beta', x.betaId);
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('set sv_voice "0"\n') }]);
await activateFiles('beta', x.betaId);
expect(await readFile(x.path, 'utf8')).toContain('set sv_voice "0"');
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
expect(await readFile(x.path, 'utf8')).toBe('original config');
});
it.each(['stale', 'identical', 'empty', 'unbundled', 'duplicate', 'closed', 'deleted', 'interrupted', 'activating'])('rejects %s updates without saving a revision', async reason => {
const x = await setup();
const upload = { scriptId: x.script.id, content: Buffer.from('changed') };
if (reason === 'identical') upload.content = Buffer.from(x.version.content);
if (reason === 'empty') upload.content = Buffer.alloc(0);
if (reason === 'unbundled') upload.scriptId = 999;
if (reason === 'closed') x.db.prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(x.betaId);
if (reason === 'deleted') x.db.prepare('UPDATE beta_servers SET server_deleted_at = ? WHERE beta_id = ?').run(new Date().toISOString(), x.betaId);
if (reason === 'interrupted') x.db.prepare("UPDATE staged_file_changes SET state = 'applying'").run();
if (reason === 'activating') {
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
x.db.prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(job.id);
}
await expect(x.reviseBeta(x.betaId, reason === 'stale' ? 2 : 1, reason === 'duplicate' ? [upload, upload] : [upload])).rejects.toThrow();
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
expect(x.listVersions(x.script.id)).toHaveLength(1);
expect(x.listBetaScripts(x.betaId)[0].script_version_id).toBe(x.version.id);
});
it('keeps the new revision through reload, and uses its deadline rather than the beta opening time', async () => {
const x = await setup();
x.db.prepare('UPDATE betas SET opened_at = ?').run('2000-01-01T00:00:00Z');
x.db.prepare("UPDATE beta_test_plans SET outcome = 'inconclusive'").run();
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
const { plannedBetaTick } = await import('../../src/lib/server/pipeline/beta-plan-runner.js');
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
await plannedBetaTick(); expect(testView(x.betaId)?.outcome).toBe('active');
const { GET } = await import('../../src/routes/api/betas/[id]/+server.js');
const body = await (await GET({ params: { id: String(x.betaId) } })).json();
expect(body.revision.revision).toBe(2); expect(body.revisions).toHaveLength(2);
});
it('supports legacy betas without a test plan', async () => {
const x = await setup({ legacy: true });
await activateFiles('beta', x.betaId);
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', match_seen_at: null, healthy_at: null });
expect(x.db.prepare('SELECT * FROM beta_test_plans').all()).toHaveLength(0);
});
it('rolls back the entire revision when a later upload fails validation', async () => {
const x = await setup();
const overlay = x.createScript({ name: 'Overlay', kind: 'config', content: Buffer.from('set sv_voice "1"\n'), scopeType: 'game', scopeValue: 't5' });
x.db.prepare("INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, 'config')")
.run(x.betaId, overlay.script.id, overlay.version.id);
await expect(x.reviseBeta(x.betaId, 1, [
{ scriptId: x.script.id, content: Buffer.from('valid new version') },
{ scriptId: overlay.script.id, content: Buffer.from('invalid config') }
])).rejects.toThrow('at least one');
expect(x.listVersions(x.script.id)).toHaveLength(1);
expect(x.listVersions(overlay.script.id)).toHaveLength(1);
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
expect(x.listBetaScripts(x.betaId).find(file => file.script_id === x.script.id)?.script_version_id).toBe(x.version.id);
});
it('accepts reviewed multipart updates and downloads immutable revision files', async () => {
const x = await setup();
const { POST } = await import('../../src/routes/api/betas/[id]/revisions/+server.js');
const form = new FormData(); form.set('expectedRevision', '1'); form.set('notes', 'Fix scoring');
form.set(`file:${x.script.id}`, new File(['changed'], 'revision.gsc'));
const post = () => POST({ params: { id: String(x.betaId) }, request: new Request('http://test/revisions', { method: 'POST', body: form }) });
expect((await post()).status).toBe(200); expect((await post()).status).toBe(409);
const { GET } = await import('../../src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.js');
const download = await GET({ params: { id: String(x.betaId), revision: '1', scriptId: String(x.script.id) } });
expect(await download.text()).toBe('revision one');
});
});
+35
View File
@@ -1276,6 +1276,41 @@ describe('native provisioning cancellation and rollback', () => {
}); });
describe('cancelled native mutation bookkeeping', () => { describe('cancelled native mutation bookkeeping', () => {
it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta, reviseBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
const { reportHostOperation, getHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { script, version } = createScript({ name: 'Revised beta', kind: 'script', targetRelPath: 'revision.gsc',
content: Buffer.from('revision one'), scopeType: 'game', scopeValue: 't5' });
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
servers: [job.name], waitForHealth: true, waitForMatch: false });
async function activate(before: string | null) {
const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
await advance();
const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?').get(restart.id) as { stage_operation_id: string };
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ id: number; path: string }>;
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
await advance();
// A dispatched activation must settle before revision submission.
await expect(reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('premature update') }])).rejects.toThrow();
const activation = (await operationFor(restart.id))!;
reportHostOperation(hostId, activation.id, { state: 'completed', result: {
before: files.map(file => ({ ...file, contentBase64: before === null ? null : Buffer.from(before).toString('base64') }))
} });
await advance();
expect((await detail(restart.id)).state).toBe('done');
}
await activate(original);
await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]);
await activate('revision one');
const rollback = await rollbackBeta(betaId);
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
.toEqual({ content: original === null ? null : Buffer.from(original) });
});
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => { it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
const job = await queue(); const job = await queue();
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');