Allow revising active betas with fresh tests and history
This commit is contained in:
+35
-3
@@ -197,8 +197,8 @@ Set wave size to 1 for a canary, a larger number for batches, or 0 for all
|
||||
selected servers. Later waves receive files after the previous waves pass.
|
||||
Health-outage budget violations on any tested server block promotion.
|
||||
|
||||
The test window starts when the beta opens, including time waiting for empty
|
||||
servers. Missing evidence at the deadline produces **inconclusive**.
|
||||
The test window starts when the beta opens or a revision is submitted, including
|
||||
time waiting for empty servers. Missing evidence at the deadline produces **inconclusive**.
|
||||
Health monitoring continues after a pass until promotion.
|
||||
|
||||
For failures, select hold as failed, pause for investigation, or automatic
|
||||
@@ -208,7 +208,7 @@ The manual option holds those changes in a rollback campaign until you select
|
||||
**Resume when empty**. Promotion is always manual.
|
||||
|
||||
Pause holds evaluation and wave advancement. Resume retains completed progress and the
|
||||
original deadline, but starts new timing intervals. A fresh attempt resets
|
||||
current revision's deadline, but starts new timing intervals. A fresh attempt resets
|
||||
progress and restarts deployed servers when empty. Recovery cycles repeat
|
||||
gameplay milestones and checklist sign-offs while carrying health outages forward.
|
||||
|
||||
@@ -217,6 +217,32 @@ recent 200 evidence entries. On upgrading from the old reporter protocol,
|
||||
active progress resets. Unsupported checks in existing plans and saved presets
|
||||
become manual review items; they cannot silently pass.
|
||||
|
||||
## Updating an active beta
|
||||
|
||||
Choose **Update beta** on the beta detail page, upload replacements for existing
|
||||
bundle files, and review the changes and selected servers before choosing
|
||||
**Deploy revision**. Text files show a diff when both files are small enough;
|
||||
binary and large files show their checksums. Files must actually change.
|
||||
|
||||
The beta keeps its ID, servers, scope and test policy. Each revision creates new
|
||||
file versions, clears current test progress and checklist sign-offs, repeats the
|
||||
configured waves, and starts a fresh testing window at submission. Files install
|
||||
and servers restart through the normal empty-server and capacity gates. Promotion
|
||||
requires the new revision to pass. Pause and retry retain the current revision's
|
||||
deadline; they do not create a revision.
|
||||
|
||||
Revision history retains downloadable bundle files and previous test results,
|
||||
including the last 200 evidence entries shown at replacement. All underlying
|
||||
attempts and evidence remain in SQLite, separated by revision. Rejecting the beta
|
||||
restores the original pre-beta files, even after multiple revisions or while a
|
||||
new revision is waiting to deploy. Files originally absent are removed.
|
||||
|
||||
Updates are available until promotion or rollback, including paused, failed and
|
||||
expired tests. An activation already in progress must settle first; interrupted
|
||||
file writes and deleted participants must be resolved before an update. Pending,
|
||||
unsubmitted restarts are superseded. Adding/removing files, changing targets or
|
||||
changing the policy still requires a new beta.
|
||||
|
||||
## API and storage
|
||||
|
||||
`POST /api/betas` accepts `testPlan`.
|
||||
@@ -228,6 +254,12 @@ recent evidence under `test`.
|
||||
`GET/POST /api/betas/presets` lists/saves reusable plans. Saved presets are
|
||||
copied into a beta; later edits do not change an active plan.
|
||||
|
||||
`POST /api/betas/:id/revisions` accepts multipart `expectedRevision`, optional
|
||||
`notes`, and replacement uploads named `file:<scriptId>`. A stale revision is
|
||||
rejected. `GET /api/betas/:id` includes the current `revision` and `revisions`
|
||||
history; `GET /api/betas/:id/revisions/:revision/files/:scriptId` downloads an
|
||||
archived file (or the original pinned file before its first update).
|
||||
|
||||
Internal progress records support attempts and recovery cycles. The reporter
|
||||
does not know about or supply these records. Legacy betas without a test plan
|
||||
also use `game_end` at round 2 or later for their match-completion gate.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import type { BetaTestView } from './beta-plan.js';
|
||||
|
||||
export interface BetaRevisionFile {
|
||||
script_id: number; name: string; kind: string; path: string;
|
||||
version_id: number; version: number; release_version: string | null; sha256: string;
|
||||
}
|
||||
export interface BetaRevision {
|
||||
revision: number; started_at: string; notes: string | null;
|
||||
files: BetaRevisionFile[];
|
||||
results: { state: string; test: BetaTestView | null; servers: Array<{ server: string; stage: string; error: string | null }> } | null;
|
||||
}
|
||||
@@ -347,6 +347,7 @@ export function db(): Database.Database {
|
||||
migrateTable(handle, 'scripts', SCRIPTS_COLUMNS);
|
||||
migrateTable(handle, 'script_versions', SCRIPT_VERSIONS_COLUMNS);
|
||||
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
|
||||
migrateTable(handle, 'beta_test_runs', [{ name: 'revision', ddl: 'ALTER TABLE beta_test_runs ADD COLUMN revision INTEGER NOT NULL DEFAULT 1' }]);
|
||||
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
|
||||
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
|
||||
migrateTable(handle, 'server_launch_collections', [
|
||||
|
||||
@@ -180,6 +180,32 @@ CREATE TABLE IF NOT EXISTS betas (
|
||||
CREATE INDEX IF NOT EXISTS idx_betas_state ON betas(state);
|
||||
CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id);
|
||||
|
||||
-- Revisions keep the beta identity and original rollback images. Archived
|
||||
-- files are independent copies so version cleanup cannot erase this history.
|
||||
CREATE TABLE IF NOT EXISTS beta_revisions (
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
revision INTEGER NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
results_json TEXT,
|
||||
PRIMARY KEY (beta_id, revision)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS beta_revision_files (
|
||||
beta_id INTEGER NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
script_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
version_id INTEGER NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
release_version TEXT,
|
||||
sha256 TEXT NOT NULL,
|
||||
content BLOB NOT NULL,
|
||||
PRIMARY KEY (beta_id, revision, script_id),
|
||||
FOREIGN KEY (beta_id, revision) REFERENCES beta_revisions(beta_id, revision) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS beta_servers (
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
server TEXT NOT NULL,
|
||||
@@ -208,6 +234,7 @@ CREATE TABLE IF NOT EXISTS beta_test_plans (
|
||||
CREATE TABLE IF NOT EXISTS beta_test_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
server TEXT NOT NULL,
|
||||
phase INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { db } from '../db/index.js';
|
||||
import type { BetaRevision } from '../../beta-revision.js';
|
||||
|
||||
export function currentBetaRevision(id: number): { revision: number; started_at: string } {
|
||||
return db().prepare(`SELECT revision, started_at FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC LIMIT 1`).get(id) as
|
||||
{ revision: number; started_at: string } | undefined
|
||||
?? { revision: 1, started_at: (db().prepare('SELECT opened_at FROM betas WHERE id = ?').get(id) as { opened_at: string }).opened_at };
|
||||
}
|
||||
|
||||
/** Called in the revision transaction, before replacing the pinned bundle. */
|
||||
export function archiveBetaRevision(id: number, revision: number, startedAt: string, notes: string | null): void {
|
||||
db().prepare('INSERT OR IGNORE INTO beta_revisions (beta_id, revision, started_at, notes) VALUES (?, ?, ?, ?)')
|
||||
.run(id, revision, startedAt, notes);
|
||||
db().prepare(`INSERT OR IGNORE INTO beta_revision_files
|
||||
(beta_id, revision, script_id, name, kind, path, version_id, version, release_version, sha256, content)
|
||||
SELECT bs.beta_id, ?, s.id, s.name, s.kind, s.target_rel_path, v.id, v.version, v.release_version, v.sha256, v.content
|
||||
FROM beta_scripts bs JOIN scripts s ON s.id = bs.script_id JOIN script_versions v ON v.id = bs.script_version_id
|
||||
WHERE bs.beta_id = ?`).run(revision, id);
|
||||
}
|
||||
|
||||
export function betaRevisionHistory(id: number): BetaRevision[] {
|
||||
const rows = db().prepare('SELECT revision, started_at, notes, results_json FROM beta_revisions WHERE beta_id = ? ORDER BY revision DESC')
|
||||
.all(id) as Array<{ revision: number; started_at: string; notes: string | null; results_json: string | null }>;
|
||||
return rows.map(({ results_json, ...row }) => ({ ...row,
|
||||
results: results_json ? JSON.parse(results_json) : null,
|
||||
files: db().prepare(`SELECT script_id, name, kind, path, version_id, version, release_version, sha256
|
||||
FROM beta_revision_files WHERE beta_id = ? AND revision = ? ORDER BY script_id`).all(id, row.revision) as BetaRevision['files']
|
||||
}));
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { parseBetaPlan, type BetaPlan, type BetaTestView } from '../../beta-plan
|
||||
import type { GameTelemetry } from '../../telemetry.js';
|
||||
import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type TestMetrics } from './beta-evaluation.js';
|
||||
import { ONLINE_MAX_MS } from '../../server-freshness.js';
|
||||
import { currentBetaRevision } from './beta-revisions.js';
|
||||
|
||||
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
|
||||
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
|
||||
@@ -33,8 +34,8 @@ export async function prepareRun(betaId: number, server: string, phase = 0): Pro
|
||||
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
|
||||
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()));
|
||||
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics, revision) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
||||
.run(randomUUID(), betaId, server, phase, nowIso(), JSON.stringify(emptyMetrics()), currentBetaRevision(betaId).revision);
|
||||
})();
|
||||
return currentRun(betaId, server)!;
|
||||
}
|
||||
@@ -97,5 +98,5 @@ export function testView(betaId: number): BetaTestView | null {
|
||||
}),
|
||||
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
|
||||
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
|
||||
WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
|
||||
WHERE r.beta_id = ? AND r.revision = ? ORDER BY e.id DESC LIMIT 200`).all(betaId, currentBetaRevision(betaId).revision) as BetaTestView['evidence'] };
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
|
||||
import { storedPlan } from './beta-tests.js';
|
||||
import { storedPlan, testView, activity, setOutcome } from './beta-tests.js';
|
||||
import { archiveBetaRevision, currentBetaRevision } from './beta-revisions.js';
|
||||
import type { BetaPromotionPreview } from '../../beta-promotion.js';
|
||||
|
||||
const operations = new Set<number>();
|
||||
@@ -28,17 +29,18 @@ import {
|
||||
listVersions,
|
||||
serversMatchingScope,
|
||||
setScope,
|
||||
uploadNewVersion,
|
||||
type ScopeType,
|
||||
type ScriptRow,
|
||||
type ScriptVersionRow
|
||||
} from './scripts.js';
|
||||
import { composeServerCfg } from './config-compose.js';
|
||||
import { getServerScript, removeScript } from './inventory.js';
|
||||
import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js';
|
||||
import { cancelStagedFiles, isFileApplied, queueFileChange, stagedFileChanges } from './staged-files.js';
|
||||
import { emit } from '../pipeline/bus.js';
|
||||
import { cancelProvisioningJob } from '../provisioning/cancellation.js';
|
||||
import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js';
|
||||
import { requestRestart } from '../provisioning/requests.js';
|
||||
import { requestRestart, releaseRestartRequest } from '../provisioning/requests.js';
|
||||
import type { BetaStage } from '../pipeline/bus.js';
|
||||
|
||||
// Whether an artifact writes a dedicated on-disk file (script/plugin) or
|
||||
@@ -497,6 +499,124 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
};
|
||||
}
|
||||
|
||||
export interface BetaRevisionUpload { scriptId: number; content: Buffer; releaseVersion?: string }
|
||||
|
||||
/** Replace existing bundle members; commit files, deadline and evidence reset together. */
|
||||
export async function reviseBeta(betaId: number, expectedRevision: number, uploads: BetaRevisionUpload[], notes = ''): Promise<{ revision: number }> {
|
||||
// Drain a runner already observing the previous revision before resetting it.
|
||||
const { betaTick } = await import('../pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
return exclusive(betaId, async () => {
|
||||
const beta = getBeta(betaId);
|
||||
if (!beta) throw new BetaError('Beta not found', 'no-such-beta');
|
||||
if (!['running', 'ready-promote'].includes(beta.state)) throw new BetaError('Beta is closed', 'invalid-state');
|
||||
const current = currentBetaRevision(betaId);
|
||||
if (current.revision !== expectedRevision) throw new BetaError('The beta changed. Reload and review the current revision.', 'preview-changed');
|
||||
if (!uploads.length || new Set(uploads.map(file => file.scriptId)).size !== uploads.length) {
|
||||
throw new BetaError('Choose at least one replacement, once per file.', 'invalid-state');
|
||||
}
|
||||
if (notes.length > 1000) throw new BetaError('Change notes must be at most 1000 characters.', 'invalid-state');
|
||||
const bundle = listBetaScripts(betaId);
|
||||
for (const file of uploads) {
|
||||
const member = bundle.find(row => row.script_id === file.scriptId);
|
||||
if (!member) throw new BetaError('Updates can only replace files already in this beta.', 'invalid-state');
|
||||
if (!file.content.length) throw new BetaError('Choose non-empty replacement files.', 'invalid-state');
|
||||
if (getVersion(member.script_version_id)!.sha256 === sha256Buffer(file.content)) {
|
||||
throw new BetaError(`${getScript(file.scriptId)!.name}: the replacement is identical to the current version.`, 'invalid-state');
|
||||
}
|
||||
}
|
||||
const servers = listBetaServers(betaId);
|
||||
if (!servers.length || servers.some(server => !isActiveBetaServer(betaId, server.server))) {
|
||||
throw new BetaError('An original beta server is missing. Roll back and open a new beta.', 'invalid-state');
|
||||
}
|
||||
ensureHostSchema(db());
|
||||
assertNativeActivationSettled(betaId);
|
||||
if (stagedFileChanges('beta', String(betaId)).some(row => row.state === 'applying')) {
|
||||
throw new BetaError('A file write was interrupted. Resolve it before updating the beta.', 'conflict');
|
||||
}
|
||||
const jobs = db().prepare(`SELECT DISTINCT d.id, d.state, d.submitted_at, n.operation_id FROM server_deployments d
|
||||
JOIN provisioning_sources s ON s.operation_id = d.id LEFT JOIN native_provisioning n ON n.deployment_id = d.id
|
||||
WHERE s.kind = 'beta' AND s.source_id = ?`).all(String(betaId)) as
|
||||
Array<{ id: string; state: string; submitted_at: string | null; operation_id: string | null }>;
|
||||
if (jobs.some(job => ['running', 'awaiting-health'].includes(job.state)
|
||||
|| (job.state === 'interrupted' && servers.some(server => server.provisioning_id === job.id))
|
||||
|| (['waiting', 'queued'].includes(job.state) && (job.submitted_at || job.operation_id)))) {
|
||||
throw new BetaError('Wait for the current activation and recovery to finish before updating this beta.', 'conflict');
|
||||
}
|
||||
const stored = storedPlan(betaId);
|
||||
const plan = stored ? parseBetaPlan(JSON.parse(stored.plan)) : null;
|
||||
const revision = current.revision + 1;
|
||||
const at = nowIso();
|
||||
db().transaction(() => {
|
||||
archiveBetaRevision(betaId, current.revision, current.started_at, null);
|
||||
db().prepare('UPDATE beta_revisions SET results_json = ? WHERE beta_id = ? AND revision = ?')
|
||||
.run(JSON.stringify({ state: beta.state, test: testView(betaId), servers }), betaId, current.revision);
|
||||
for (const upload of uploads) {
|
||||
const version = uploadNewVersion(upload.scriptId, upload.content, notes.trim() || undefined, upload.releaseVersion);
|
||||
db().prepare('UPDATE beta_scripts SET script_version_id = ? WHERE beta_id = ? AND script_id = ?')
|
||||
.run(version.id, betaId, upload.scriptId);
|
||||
db().prepare('UPDATE betas SET script_version_id = ? WHERE id = ? AND script_id = ?')
|
||||
.run(version.id, betaId, upload.scriptId);
|
||||
}
|
||||
// Preflight all targets before cancelling work. Errors roll back the
|
||||
// uploads and history as well as the beta changes.
|
||||
const changes = new Map(servers.map(server => [server.server, revisionFileChanges(betaId, server.server)]));
|
||||
// Betas created before staged activation may only have deployed_at
|
||||
// and snapshots. Record their first writes before replacing staging.
|
||||
for (const server of servers.filter(server => server.deployed_at)) {
|
||||
for (const change of changes.get(server.server)!) {
|
||||
if (isFileApplied('beta', String(betaId), server.server, change.kind, change.relPath) === null) {
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
|
||||
.run(betaId, server.server, change.relPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const job of jobs.filter(job => ['waiting', 'queued'].includes(job.state))) {
|
||||
cancelProvisioningJob(job.id, `Superseded by beta #${betaId} revision ${revision}`);
|
||||
}
|
||||
for (const server of servers) releaseRestartRequest(server.server, { kind: 'beta', id: String(betaId) });
|
||||
// New IDs prevent historical host receipts from resolving to a newer
|
||||
// revision's desired bytes. Original before images stay intact.
|
||||
db().prepare("DELETE FROM staged_file_changes WHERE source_kind = 'beta' AND source_id = ?").run(String(betaId));
|
||||
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(at, betaId);
|
||||
db().prepare(`UPDATE beta_servers SET stage = 'queued', error = NULL, deployed_at = NULL,
|
||||
restarted_at = NULL, healthy_at = NULL, match_seen_at = NULL, updated_at = ? WHERE beta_id = ?`).run(at, betaId);
|
||||
db().prepare("UPDATE betas SET state = 'running' WHERE id = ?").run(betaId);
|
||||
setOutcome(betaId, 'active', null);
|
||||
archiveBetaRevision(betaId, revision, at, notes.trim() || null);
|
||||
// The original first wave follows selection order; later waves are
|
||||
// ordered by the runner. Preserve that canary selection on revision.
|
||||
const firstWave = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ? ORDER BY rowid LIMIT ?')
|
||||
.all(betaId, plan?.waveSize || servers.length) as { server: string }[];
|
||||
for (const server of firstWave) {
|
||||
for (const change of changes.get(server.server)!) queueFileChange(change);
|
||||
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty' WHERE beta_id = ? AND server = ?").run(betaId, server.server);
|
||||
}
|
||||
activity(betaId, `Revision ${revision} submitted; fresh tests and testing window${notes.trim() ? `: ${notes.trim()}` : ''}`);
|
||||
})();
|
||||
emit({ type: 'beta-state', betaId, state: 'running' });
|
||||
return { revision };
|
||||
});
|
||||
}
|
||||
|
||||
function revisionFileChanges(betaId: number, name: string): Parameters<typeof queueFileChange>[0][] {
|
||||
const resolved = listBetaScripts(betaId).map(row => ({ script: getScript(row.script_id)!, version: getVersion(row.script_version_id)! }));
|
||||
const server = getServer(name)!;
|
||||
const changes: Parameters<typeof queueFileChange>[0][] = resolved.filter(row => !isCollapsingKind(row.script.kind))
|
||||
.map(({ script, version }) => ({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: script.kind,
|
||||
relPath: script.target_rel_path, content: version.content, scriptVersionId: version.id, inventorySource: `beta:${betaId}` }));
|
||||
if (resolved.some(row => isCollapsingKind(row.script.kind))) {
|
||||
const composed = composeCfgForServer(name, server.game, server.map, {
|
||||
inBetaBaseline: resolved.find(row => row.script.kind === 'baseline')?.version,
|
||||
inBetaOverlays: resolved.filter(row => row.script.kind === 'config')
|
||||
});
|
||||
if (!composed) throw new BetaError(`No baseline available for ${name}.`, 'invalid-state');
|
||||
changes.push({ sourceKind: 'beta', sourceId: String(betaId), server: name, kind: 'config', relPath: configFileOf(name),
|
||||
content: Buffer.from(composed.text), inventorySource: `beta:${betaId}` });
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
|
||||
/** Apply a reserved wave only when its predecessors have passed. */
|
||||
export async function deployBetaServers(betaId: number, names: string[]): Promise<void> {
|
||||
return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
|
||||
@@ -620,6 +740,8 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
|
||||
const restorations: Array<{ server: string; kind: ArtifactKind; relPath: string; content: Buffer | null; versionId: number | null }> = [];
|
||||
const changed = db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?');
|
||||
const applied = (bs: BetaServerRow, kind: ArtifactKind, path: string) => {
|
||||
// A queued revision may supersede an applied one before its own activation.
|
||||
if (changed.get(betaId, bs.server, path)) return true;
|
||||
const staged = isFileApplied('beta', String(betaId), bs.server, kind, path);
|
||||
if (staged !== null) return staged;
|
||||
// Existing betas predating staging wrote their files before deployed_at.
|
||||
|
||||
@@ -80,6 +80,21 @@ function readExisting(path: string): Buffer | null {
|
||||
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err; }
|
||||
}
|
||||
|
||||
/** The first activation owns the rollback image, including an absent file. */
|
||||
export function captureBetaBeforeImage(row: StagedFileChange, before: Buffer | null): void {
|
||||
if (db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?')
|
||||
.get(row.source_id, row.server, row.rel_path)) return;
|
||||
const prior = getServerScript(row.server, row.rel_path);
|
||||
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
|
||||
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
|
||||
.run(row.source_id, row.server, row.rel_path, before, sha256Buffer(before), before.length);
|
||||
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
|
||||
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
|
||||
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, row.server, row.rel_path, row.kind);
|
||||
db().prepare('INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
|
||||
.run(row.source_id, row.server, row.rel_path);
|
||||
}
|
||||
|
||||
/** Only called after occupancy checks and reservation of the restart slot.
|
||||
* Synchronous file replacement keeps cancellation, new requests, and deletion
|
||||
* from interleaving between the final gates and dispatching the restart.
|
||||
@@ -111,17 +126,9 @@ export function applyStagedFilesForOperation(operationId: string, server: string
|
||||
for (const { row, path } of plans) {
|
||||
// Earlier coalesced campaigns may have just updated the same path.
|
||||
const before = readExisting(path);
|
||||
const prior = getServerScript(server, row.rel_path);
|
||||
db().transaction(() => {
|
||||
if (row.source_kind === 'beta') {
|
||||
if (before !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots
|
||||
(beta_id, server, rel_path, prior_content, prior_sha256, prior_size) VALUES (?, ?, ?, ?, ?, ?)`)
|
||||
.run(row.source_id, server, row.rel_path, before, sha256Buffer(before), before.length);
|
||||
db().prepare(`UPDATE beta_server_scripts SET had_original = ?, prior_version_id = ? WHERE beta_id = ? AND server = ?
|
||||
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path = ? AND kind = ?)`)
|
||||
.run(before === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, server, row.rel_path, row.kind);
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)')
|
||||
.run(row.source_id, server, row.rel_path);
|
||||
captureBetaBeforeImage(row, before);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare(`UPDATE deployment_files SET
|
||||
verb = ?, content_before = ?, sha256_before = ? WHERE id = ?`)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
|
||||
import { currentBetaRevision } from '../fleet/beta-revisions.js';
|
||||
import { betaOperationInFlight, deployBetaServers, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
|
||||
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
|
||||
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
|
||||
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
|
||||
@@ -64,7 +65,6 @@ async function tick(): Promise<void> {
|
||||
async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
const id = stored.beta_id;
|
||||
const plan = parseBetaPlan(JSON.parse(stored.plan));
|
||||
const beta = getBeta(id)!;
|
||||
const servers = listBetaServers(id);
|
||||
// Preserve the original coverage and evidence. A replacement with the same
|
||||
// name cannot complete a test that began on a different server.
|
||||
@@ -74,7 +74,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
|
||||
return;
|
||||
}
|
||||
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
|
||||
if (stored.outcome !== 'passed' && Date.now() - Date.parse(currentBetaRevision(id).started_at) >= plan.timeoutHours * 3_600_000) {
|
||||
betaState(id, false);
|
||||
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
|
||||
return;
|
||||
|
||||
@@ -22,9 +22,9 @@ import { fleetSettings } from '../settings.js';
|
||||
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
||||
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
|
||||
import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js';
|
||||
import { recordScript, removeScript } from '../fleet/inventory.js';
|
||||
import { sha256Buffer } from '../fleet/files.js';
|
||||
import type { StagedFileChange } from '../fleet/staged-files.js';
|
||||
import { captureBetaBeforeImage, type StagedFileChange } from '../fleet/staged-files.js';
|
||||
import type { LaunchPreview } from '../../deployment-plan.js';
|
||||
import type { ProvisioningAction } from '../../provisioning.js';
|
||||
import { hostAgentUpdatesEnabled } from '../../hosts.js';
|
||||
@@ -227,13 +227,8 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
|
||||
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
|
||||
if (!row || row.state === 'applied') continue;
|
||||
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
|
||||
const prior = getServerScript(job.server, row.rel_path);
|
||||
if (row.source_kind === 'beta') {
|
||||
if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`)
|
||||
.run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length);
|
||||
db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=?
|
||||
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind);
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path);
|
||||
captureBetaBeforeImage(row, bytes);
|
||||
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
<script lang="ts">
|
||||
import Dialog from './primitives/Dialog.svelte';
|
||||
import Button from './primitives/Button.svelte';
|
||||
import DiffViewer from './DiffViewer.svelte';
|
||||
import ServerIdentity from './ServerIdentity.svelte';
|
||||
import { artifactVersionLabel } from '$lib/artifact-version.js';
|
||||
|
||||
interface FileEntry { id: number; name: string; kind: string; target_rel_path: string;
|
||||
version: { version: number; release_version: string | null; sha256: string } | null }
|
||||
let { open = $bindable(false), betaId, revision, files, servers, timeoutHours, onUpdate }:
|
||||
{ open?: boolean; betaId: number; revision: number; files: FileEntry[]; servers: string[];
|
||||
timeoutHours?: number; onUpdate: () => Promise<void> } = $props();
|
||||
interface Replacement { file: File; sha: string; before: string | null; after: string | null }
|
||||
let replacements = $state<Record<number, Replacement>>({});
|
||||
let notes = $state(''), error = $state(''), reviewing = $state(false), busy = $state(false);
|
||||
const count = $derived(Object.keys(replacements).length);
|
||||
$effect(() => { if (open) { replacements = {}; notes = ''; error = ''; reviewing = false; } });
|
||||
function fileUrl(scriptId: number) { return `/api/betas/${betaId}/revisions/${revision}/files/${scriptId}`; }
|
||||
async function choose(entry: FileEntry, file?: File) {
|
||||
delete replacements[entry.id]; error = '';
|
||||
if (!file) return;
|
||||
busy = true;
|
||||
try {
|
||||
if (!file.size) throw new Error('Choose a non-empty file.');
|
||||
const bytes = await file.arrayBuffer();
|
||||
const inspection = new FormData(); inspection.set('content', file);
|
||||
const inspected = await fetch('/api/scripts/inspect', { method: 'POST', body: inspection });
|
||||
const inspectionBody = await inspected.json();
|
||||
if (!inspected.ok) throw new Error(inspectionBody.error ?? 'Could not inspect replacement file.');
|
||||
const sha = inspectionBody.sha256 as string;
|
||||
if (sha === entry.version?.sha256) throw new Error(`${entry.name}: this file is identical to the current version.`);
|
||||
let before: string | null = null, after: string | null = null;
|
||||
if (entry.kind !== 'plugin' && bytes.byteLength <= 128 * 1024) {
|
||||
const response = await fetch(fileUrl(entry.id));
|
||||
if (!response.ok) throw new Error('Could not load the current file for review. Reload the beta and try again.');
|
||||
const old = await response.arrayBuffer();
|
||||
if (old.byteLength <= 128 * 1024 && !new Uint8Array(old).includes(0) && !new Uint8Array(bytes).includes(0)) {
|
||||
before = new TextDecoder().decode(old); after = new TextDecoder().decode(bytes);
|
||||
}
|
||||
}
|
||||
replacements[entry.id] = { file, sha, before, after };
|
||||
} catch (err) { error = (err as Error).message; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
async function deploy() {
|
||||
busy = true; error = '';
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.set('expectedRevision', String(revision)); form.set('notes', notes);
|
||||
for (const [scriptId, replacement] of Object.entries(replacements)) form.set(`file:${scriptId}`, replacement.file);
|
||||
const response = await fetch(`/api/betas/${betaId}/revisions`, { method: 'POST', body: form });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not update beta.');
|
||||
open = false; await onUpdate();
|
||||
} catch (err) { error = (err as Error).message; }
|
||||
finally { busy = false; }
|
||||
}
|
||||
</script>
|
||||
|
||||
<Dialog bind:open title={reviewing ? `Review revision ${revision + 1}` : 'Update beta'}
|
||||
description={`Replace files in beta #${betaId} and repeat its tests on the same servers.`} {busy} wide>
|
||||
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
|
||||
{#if !reviewing}
|
||||
<p>Choose replacements for the files you changed.</p>
|
||||
<div class="files">{#each files as entry}
|
||||
<label><strong>{entry.name} · {artifactVersionLabel(entry.version)}</strong><code>{entry.target_rel_path}</code>
|
||||
<input type="file" disabled={busy} onchange={event => choose(entry, event.currentTarget.files?.[0])} />
|
||||
{#if replacements[entry.id]}<span>Replacement ready for review</span>{/if}
|
||||
</label>
|
||||
{/each}</div>
|
||||
<label class="notes">Change note (optional)<textarea bind:value={notes} maxlength="1000" rows="3" disabled={busy}></textarea></label>
|
||||
{:else}
|
||||
<p><strong>{count} file{count === 1 ? '' : 's'} changed.</strong> All {servers.length} selected servers will repeat the test policy, using its existing wave order. Each server installs the bundle and restarts when empty.</p>
|
||||
<p>Previous results move to revision history. Promotion requires this revision to pass.
|
||||
{#if timeoutHours}A fresh {timeoutHours}-hour test window starts when you deploy, including time waiting for empty servers.{/if}
|
||||
Rejecting the beta still restores the files from before revision 1.</p>
|
||||
<details open><summary>Servers ({servers.length})</summary><ul>{#each servers as name}<li><ServerIdentity {name}/></li>{/each}</ul></details>
|
||||
{#each files.filter(entry => replacements[entry.id]) as entry}
|
||||
{@const replacement = replacements[entry.id]}
|
||||
<section><h3>{entry.name}</h3><p><code>{entry.target_rel_path}</code> ← {replacement.file.name} ({replacement.file.size.toLocaleString()} bytes)</p>
|
||||
<p class="hash">SHA {entry.version?.sha256.slice(0,12)} → {replacement.sha.slice(0,12)}</p>
|
||||
{#if replacement.after !== null}<DiffViewer before={replacement.before} after={replacement.after} verb="updated" relPath={entry.target_rel_path}/>
|
||||
{:else}<p>Binary or large file: review the uploaded file and checksum.</p>{/if}
|
||||
</section>
|
||||
{/each}
|
||||
{#if notes}<p>Change note: {notes}</p>{/if}
|
||||
{/if}
|
||||
{#snippet actions()}
|
||||
<Button onclick={() => open = false} disabled={busy}>Cancel</Button>
|
||||
{#if reviewing}<Button onclick={() => reviewing = false} disabled={busy}>Back</Button><Button variant="primary" onclick={deploy} disabled={busy} loading={busy}>Deploy revision {revision + 1}</Button>
|
||||
{:else}<Button variant="primary" onclick={() => reviewing = true} disabled={busy || !count}>Review update</Button>{/if}
|
||||
{/snippet}
|
||||
</Dialog>
|
||||
|
||||
<style>
|
||||
.files, label { display:grid; gap:12px; } .files { margin:20px 0; }
|
||||
.files label { padding:16px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); }
|
||||
p { line-height:1.6; } code, .hash { overflow-wrap:anywhere; } .hash, label span { color:var(--text-secondary); font-size:13px; }
|
||||
li { margin:8px 0; } textarea { width:100%; } section { margin-top:24px; }
|
||||
</style>
|
||||
@@ -50,7 +50,7 @@
|
||||
{#if ['paused','failed'].includes(test.outcome) && !expired}<Button size="sm" disabled={busy || !!retryBlocked} onclick={()=>action('retry')}>Start fresh attempt</Button>{/if}
|
||||
</div>{/if}
|
||||
{#if retryBlocked && ['paused','failed'].includes(test.outcome)}<p>{retryBlocked}</p>{/if}
|
||||
{#if expired}<p class="guidance">The testing window ended. Roll back this beta first, then choose “Reuse files and test plan” to open a new beta with a longer window.</p>
|
||||
{#if expired}<p class="guidance">The testing window ended. Use “Update beta” to test changed files in a fresh revision, or roll back and reuse the plan with a longer window.</p>
|
||||
{:else if test.outcome === 'paused' && !hasDeletedServers}<p class="guidance">Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.</p>{/if}
|
||||
{/if}
|
||||
{#if error}<p role="alert" class="failed">{error}</p>{/if}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
|
||||
import { kick } from '$lib/server/pipeline/worker.js';
|
||||
import { parseBetaPlan, type BetaPlan } from '$lib/beta-plan.js';
|
||||
import { storedPlan } from '$lib/server/fleet/beta-tests.js';
|
||||
import { currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
|
||||
|
||||
export async function GET({ url }: { url: URL }) {
|
||||
const stateFilter = url.searchParams.get('state') as
|
||||
@@ -38,7 +39,7 @@ export async function GET({ url }: { url: URL }) {
|
||||
...b,
|
||||
testOutcome: test?.outcome ?? null,
|
||||
testReason: test?.reason ?? null,
|
||||
deadlineAt: plan ? new Date(Date.parse(b.opened_at) + plan.timeoutHours * 3600000).toISOString() : null,
|
||||
deadlineAt: plan ? new Date(Date.parse(currentBetaRevision(b.id).started_at) + plan.timeoutHours * 3600000).toISOString() : null,
|
||||
scripts,
|
||||
servers: listBetaServers(b.id)
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ import { db } from '$lib/server/db/index.js';
|
||||
import { getBeta, listBetaServers, listBetaScripts } from '$lib/server/fleet/beta.js';
|
||||
import { getScript, getVersion } from '$lib/server/fleet/scripts.js';
|
||||
import { testView } from '$lib/server/fleet/beta-tests.js';
|
||||
import { betaRevisionHistory, currentBetaRevision } from '$lib/server/fleet/beta-revisions.js';
|
||||
|
||||
function parseId(raw: string): number | null {
|
||||
const n = Number.parseInt(raw, 10);
|
||||
@@ -37,5 +38,6 @@ export async function GET({ params }: { params: { id: string } }) {
|
||||
failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
|
||||
const rollback = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'rollback' ORDER BY started_at DESC LIMIT 1")
|
||||
.get(id) as { id: string } | undefined;
|
||||
return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null });
|
||||
return json({ beta, scripts, servers, events, test: testView(id), promotion, rollback: rollback ?? null,
|
||||
revision: currentBetaRevision(id), revisions: betaRevisionHistory(id) });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { BetaError, reviseBeta, type BetaRevisionUpload } from '$lib/server/fleet/beta.js';
|
||||
import { ScriptError } from '$lib/server/fleet/scripts.js';
|
||||
import { kick } from '$lib/server/pipeline/worker.js';
|
||||
|
||||
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
|
||||
const id = Number(params.id);
|
||||
if (!Number.isSafeInteger(id) || id < 1) return json({ ok: false, error: 'Invalid beta ID' }, { status: 400 });
|
||||
try {
|
||||
const form = await request.formData();
|
||||
const expectedRevision = Number(form.get('expectedRevision'));
|
||||
if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 1) throw new Error('A reviewed revision is required.');
|
||||
const uploads: BetaRevisionUpload[] = [];
|
||||
for (const [key, value] of form) {
|
||||
if (!key.startsWith('file:')) continue;
|
||||
const scriptId = Number(key.slice(5));
|
||||
if (!Number.isSafeInteger(scriptId) || scriptId < 1 || !(value instanceof File)) throw new Error('Invalid replacement file.');
|
||||
uploads.push({ scriptId, content: Buffer.from(await value.arrayBuffer()),
|
||||
releaseVersion: form.has(`release:${scriptId}`) ? String(form.get(`release:${scriptId}`)) : undefined });
|
||||
}
|
||||
const result = await reviseBeta(id, expectedRevision, uploads, String(form.get('notes') ?? ''));
|
||||
void kick();
|
||||
return json({ ok: true, result });
|
||||
} catch (error) {
|
||||
return json({ ok: false, error: (error as Error).message },
|
||||
{ status: error instanceof BetaError || error instanceof ScriptError ? 409 : 400 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { db } from '$lib/server/db/index.js';
|
||||
|
||||
/** Current files can be reviewed before the first revision is archived. */
|
||||
export async function GET({ params }: { params: { id: string; revision: string; scriptId: string } }) {
|
||||
const values = [params.id, params.revision, params.scriptId].map(Number);
|
||||
if (values.some(value => !Number.isSafeInteger(value) || value < 1)) return new Response('Invalid file', { status: 400 });
|
||||
const [id, revision, scriptId] = values;
|
||||
const file = db().prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = ? AND script_id = ?')
|
||||
.get(id, revision, scriptId) as { content: Buffer } | undefined
|
||||
?? (revision === 1 ? db().prepare(`SELECT v.content FROM beta_scripts bs JOIN script_versions v ON v.id = bs.script_version_id
|
||||
WHERE bs.beta_id = ? AND bs.script_id = ? AND NOT EXISTS (SELECT 1 FROM beta_revisions WHERE beta_id = ?)`)
|
||||
.get(id, scriptId, id) as { content: Buffer } | undefined : undefined);
|
||||
if (!file) return new Response('File not found', { status: 404 });
|
||||
return new Response(new Uint8Array(file.content), { headers: { 'Content-Type': 'application/octet-stream',
|
||||
'Content-Disposition': 'attachment', 'Cache-Control': 'no-store' } });
|
||||
}
|
||||
@@ -50,9 +50,8 @@ export async function POST({ params, request }: { params: { id: string }; reques
|
||||
}
|
||||
setOutcome(id, 'active', null);
|
||||
} else if (body.action === 'retry') {
|
||||
// Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
|
||||
// operators open another beta if the overall testing window has expired.
|
||||
if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
|
||||
// Fresh attempts keep history and the current revision's deadline.
|
||||
if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Update the files to start a new revision, or roll back and open a new beta with a longer deadline');
|
||||
if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
|
||||
throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
import type { ActivityEvent } from '$lib/operation-flow.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { confirmAction } from '$ui/confirm.js';
|
||||
import type { BetaRevision } from '$lib/beta-revision.js';
|
||||
import BetaRevisionDialog from '$ui/components/BetaRevisionDialog.svelte';
|
||||
import BetaTestProgress from '$ui/components/BetaTestProgress.svelte';
|
||||
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
|
||||
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
|
||||
@@ -38,10 +40,13 @@
|
||||
let acting = $state<string|null>(null), request = 0;
|
||||
let reviewOpen = $state(false), preview = $state<BetaPromotionPreview|null>(null), reviewError = $state('');
|
||||
let now = $state(Date.now());
|
||||
let revision = $state({ revision: 1, started_at: '' }), revisions = $state<BetaRevision[]>([]);
|
||||
let updateOpen = $state(false), updateRevision = $state(1), updateFiles = $state<BetaFile[]>([]);
|
||||
function openUpdate() { updateRevision = revision.revision; updateFiles = [...scripts]; updateOpen = true; }
|
||||
const id = $derived(Number($page.params.id));
|
||||
const closed = $derived(beta?.state === 'promoted' || beta?.state === 'rolled-back');
|
||||
const hasDeletedServers = $derived(servers.some(server => server.server_deleted_at));
|
||||
const deadline = $derived(beta && test ? new Date(Date.parse(beta.opened_at) + test.plan.timeoutHours * 3600000) : null);
|
||||
const deadline = $derived(beta && test ? new Date(Date.parse(revision.started_at || beta.opened_at) + test.plan.timeoutHours * 3600000) : null);
|
||||
const remaining = $derived(deadline ? Math.max(0, Math.ceil((deadline.getTime() - now) / 60000)) : 0);
|
||||
const deadlineText = $derived(remaining > 0 ? `${Math.floor(remaining / 60)}h ${remaining % 60}m remaining` : 'Testing window ended');
|
||||
const passed = $derived(test?.servers.filter(server => server.passed).length ?? servers.filter(server => server.stage === 'ready').length);
|
||||
@@ -49,7 +54,7 @@
|
||||
const waiting = $derived(ranked.filter(server => server.stage !== 'ready'));
|
||||
const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check server activation.' : rollbackCampaign ? 'Follow the rollback campaign to resume held changes or check restoration and restart progress.' : 'Reuse these files and this plan when you are ready to test again.'
|
||||
: hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.'
|
||||
: test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.'
|
||||
: test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Update the files to start a new revision, or roll back and reuse this plan.'
|
||||
: test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.'
|
||||
: test?.outcome === 'failed' ? 'Investigate the failed check before a fresh attempt, or roll back.'
|
||||
: beta?.state === 'ready-promote' ? 'All required coverage passed. Review the full promotion impact before deploying to additional servers.'
|
||||
@@ -64,11 +69,12 @@
|
||||
if (current !== request || requestedId !== id) return;
|
||||
test = body.test; beta = body.beta; scripts = body.scripts; servers = body.servers; events = body.events; promotion = body.promotion ?? null;
|
||||
rollbackCampaign = body.rollback ?? null;
|
||||
revision = body.revision; revisions = body.revisions ?? [];
|
||||
now = Date.now(); loadError = '';
|
||||
} catch (error) { if (current === request && requestedId === id) loadError = (error as Error).message; }
|
||||
finally { if (current === request && requestedId === id) loading = false; }
|
||||
}
|
||||
$effect(() => { id; beta = null; loading = true; loadError = ''; flash = ''; success = ''; reviewOpen = false; void load(); });
|
||||
$effect(() => { id; beta = null; loading = true; loadError = ''; flash = ''; success = ''; reviewOpen = false; updateOpen = false; void load(); });
|
||||
onMount(() => pollVisible(load, 5000, false));
|
||||
async function reviewPromotion() {
|
||||
reviewOpen = true; preview = null; reviewError = ''; acting = 'preview';
|
||||
@@ -129,7 +135,7 @@
|
||||
{:else if !beta && !loadError}<EmptyState title="Beta not found" description="Return to Betas to find another test." />
|
||||
{:else if beta}
|
||||
<div class="page">
|
||||
<SectionHeader title={beta.notes || `Beta #${beta.id}`} subtitle={beta.notes ? `Beta #${beta.id}` : undefined}>
|
||||
<SectionHeader title={beta.notes || `Beta #${beta.id}`} subtitle={`Beta #${beta.id} · Revision ${revision.revision}`}>
|
||||
{#snippet crumbs()}<Breadcrumbs items={[{label:'Betas',href:'/betas'},{label:`#${id}`}]} />{/snippet}
|
||||
{#snippet actions()}<StatusBadge state={test && !closed ? test.outcome : beta?.state ?? 'running'} />{/snippet}
|
||||
</SectionHeader>
|
||||
@@ -139,12 +145,13 @@
|
||||
<p>{nextAction}</p>
|
||||
{#if test?.reason}<p class="reason">{test.reason}</p>{/if}
|
||||
<div class="timing"><span>Started <time datetime={beta.opened_at}>{$timeFormat.dateTime(beta.opened_at)}</time></span>
|
||||
{#if revision.revision > 1}<span>Revision {revision.revision} started {$timeFormat.dateTime(revision.started_at)}</span>{/if}
|
||||
{#if deadline && !closed}<strong>Ends <time datetime={deadline.toISOString()}>{$timeFormat.dateTime(deadline)}</time> · {deadlineText}</strong>{/if}
|
||||
{#if beta.closed_at}<span>Finished {$timeFormat.dateTime(beta.closed_at)}</span>{/if}
|
||||
</div>
|
||||
{#if !closed && test}<p class="muted">Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline.</p>{/if}
|
||||
{#if !closed && test}<p class="muted">Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline. Deploying a revision starts a fresh window.</p>{/if}
|
||||
<div class="actions">
|
||||
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
|
||||
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button onclick={openUpdate} disabled={hasDeletedServers || acting !== null}>Update beta</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
|
||||
{#if closed}<Button href={contextHref(`/betas/new?reuse=${id}`, $page.url)}>Reuse files and test plan</Button>{/if}
|
||||
{#if promotion}<Button href={contextHref(`/campaigns/${encodeURIComponent(promotion.id)}`, $page.url)}>View campaign</Button>{/if}
|
||||
{#if rollbackCampaign}<Button href={contextHref(`/campaigns/${encodeURIComponent(rollbackCampaign.id)}`, $page.url)}>View rollback campaign</Button>{/if}
|
||||
@@ -163,10 +170,22 @@
|
||||
<details class="ui-panel"><summary>Deployment progress ({servers.length} servers)</summary><div class="server-list">
|
||||
{#each ranked as server}<div class="server-row"><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} href={contextHref(`/servers/${encodeURIComponent(server.server)}`, $page.url)}/><StatusBadge state={server.stage}/><p>Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}</p>{#if server.provisioning_id}<a href={contextHref(`/provisioning/${encodeURIComponent(server.provisioning_id)}`, $page.url)}>View provisioning</a>{/if}{#if server.error}<p class="ui-error">{server.error}</p>{/if}{#if server.stage === 'failed' && !server.server_deleted_at && !test && !closed}<Button size="sm" onclick={() => retry(server.server)} disabled={acting !== null} loading={acting === `retry:${server.server}`}>Retry server</Button>{/if}</div>{/each}
|
||||
</div></details>
|
||||
{#if revisions.length}
|
||||
<details class="ui-panel"><summary>Revision history ({revisions.length})</summary><div class="revision-list">
|
||||
{#each revisions as item}<details><summary>Revision {item.revision} · {item.revision === revision.revision ? 'Current' : item.results?.test?.outcome ?? item.results?.state ?? 'Superseded'} · {$timeFormat.dateTime(item.started_at)}</summary>
|
||||
{#if item.notes}<p>{item.notes}</p>{/if}
|
||||
<div class="file-list">{#each item.files as file}<div><a href={`/api/betas/${id}/revisions/${item.revision}/files/${file.script_id}`} download={file.path.split('/').pop()}>{file.name} · {artifactVersionLabel(file)}</a><code>{file.path}</code><span>{file.sha256.slice(0,12)}</span></div>{/each}</div>
|
||||
{#if item.results?.test}<BetaTestProgress test={item.results.test} betaId={id} closed={true} onUpdate={load} />
|
||||
{:else if item.results}<ul>{#each item.results.servers as server}<li>{server.server} · {server.stage}{server.error ? ` · ${server.error}` : ''}</li>{/each}</ul>{/if}
|
||||
</details>{/each}
|
||||
</div></details>
|
||||
{/if}
|
||||
<OperationHistory {events} active={!closed} showOverview={false}/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<BetaRevisionDialog bind:open={updateOpen} betaId={id} revision={updateRevision} files={updateFiles} servers={servers.map(server => server.server)} timeoutHours={test?.plan.timeoutHours} onUpdate={load} />
|
||||
|
||||
<Dialog bind:open={reviewOpen} title="Review promotion" description="Review the exact files and server eligibility before installing the promoted versions." busy={acting === 'promote'} wide>
|
||||
{#if acting === 'preview'}<p role="status">Preparing promotion impact…</p>{/if}
|
||||
{#if reviewError}<p class="ui-error" role="alert">{reviewError}</p>{/if}
|
||||
@@ -182,6 +201,7 @@
|
||||
</Dialog>
|
||||
|
||||
<style>
|
||||
.revision-list { display:grid; gap:20px; margin-top:16px; }
|
||||
.page { display:grid; gap:24px; }
|
||||
h2 { margin:0; font-size:18px; } p { line-height:1.6; overflow-wrap:anywhere; }
|
||||
.summary-heading,.actions,.timing { display:flex; align-items:center; gap:12px; flex-wrap:wrap; }
|
||||
|
||||
@@ -68,6 +68,30 @@ async function tick(seconds = 15, players = 4, healthy = true) {
|
||||
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
|
||||
|
||||
describe('durable beta test plans', () => {
|
||||
it('requires fresh gameplay after a revision and promotes exactly the replacement version', async () => {
|
||||
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
|
||||
await tick(); await tick();
|
||||
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
|
||||
const { getBeta, listBetaScripts, reviseBeta, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const { getScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const file = listBetaScripts(id)[0];
|
||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||
await reviseBeta(id, 1, [{ scriptId: file.script_id, content: Buffer.from('main() { /* fixed */ }') }]);
|
||||
const replacement = listBetaScripts(id)[0].script_version_id;
|
||||
expect(replacement).not.toBe(file.script_version_id);
|
||||
await postTelemetry(gameEvent('game_end')); // Still running the old revision.
|
||||
expect(testView(id)?.evidence).toHaveLength(0);
|
||||
await postTelemetry(gameEvent('heartbeat', 1, 0));
|
||||
await tick(); await tick();
|
||||
expect(getBeta(id)?.state).toBe('running');
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('game_start')); await postTelemetry(gameEvent('game_end')); await tick();
|
||||
expect(getBeta(id)?.state).toBe('ready-promote');
|
||||
await promoteBeta(id);
|
||||
expect(getScript(file.script_id)?.current_version_id).toBe(replacement);
|
||||
});
|
||||
|
||||
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
|
||||
const id = await openPlan(defaultPlan());
|
||||
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { activateFiles } from '../helpers/staged-files.js';
|
||||
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() }));
|
||||
let fixture: Fixture;
|
||||
const names = ['t5_zombie_theater_1', 't5_zombie_theater_2'];
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(names);
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
|
||||
|
||||
async function setup(options: { original?: string; waveSize?: number; kind?: 'script' | 'baseline'; legacy?: boolean } = {}) {
|
||||
const scripts = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const beta = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js');
|
||||
const kind = options.kind ?? 'script';
|
||||
const path = join(destOfKind(names[0], kind), kind === 'script' ? 'revision.gsc' : configFileOf(names[0]));
|
||||
if (options.original !== undefined) {
|
||||
await mkdir(dirname(path), { recursive: true }); await writeFile(path, options.original);
|
||||
}
|
||||
const { script, version } = scripts.createScript({ name: 'Revision test', targetRelPath: kind === 'script' ? 'revision.gsc' : undefined,
|
||||
kind, scopeType: 'game', scopeValue: 't5', content: Buffer.from(kind === 'script' ? 'revision one' : 'set sv_voice "1"\n') });
|
||||
const plan = { ...defaultPlan(), waveSize: options.waveSize ?? 0, requirements: [requirement('manual', 'review')] };
|
||||
const { betaId } = await beta.openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
||||
servers: options.waveSize ? names : [names[0]], waitForHealth: true, waitForMatch: false,
|
||||
...(options.legacy ? {} : { testPlan: plan }) });
|
||||
// Pause while arranging test state; revision submission explicitly starts testing.
|
||||
if (!options.legacy) db().prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
|
||||
return { ...scripts, ...beta, db: db(), script, version, betaId, path, plan };
|
||||
}
|
||||
|
||||
describe('beta revisions', () => {
|
||||
it('keeps the beta identity and archives files, sign-offs and evidence while resetting tests and the deadline', async () => {
|
||||
const x = await setup();
|
||||
await activateFiles('beta', x.betaId);
|
||||
const { prepareRun, saveMetrics, readMetrics, evidence, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const { currentBetaRevision, betaRevisionHistory } = await import('../../src/lib/server/fleet/beta-revisions.js');
|
||||
const run = await prepareRun(x.betaId, names[0]);
|
||||
const metrics = readMetrics(run); metrics.manual.review = 'Josh: verified'; saveMetrics(run, metrics);
|
||||
evidence(run, 'old-signoff', 'manual-sign-off', { note: 'Josh: verified' });
|
||||
x.db.prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(x.betaId);
|
||||
x.db.prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(x.betaId);
|
||||
const opened = x.getBeta(x.betaId)!.opened_at;
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.parse(opened) + 60_000);
|
||||
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('revision two') }], 'Small fix'))
|
||||
.resolves.toEqual({ revision: 2 });
|
||||
expect(x.getBeta(x.betaId)).toMatchObject({ state: 'running', opened_at: opened });
|
||||
expect(currentBetaRevision(x.betaId)).toEqual({ revision: 2, started_at: new Date().toISOString() });
|
||||
expect(testView(x.betaId)).toMatchObject({ outcome: 'active', servers: [], evidence: [] });
|
||||
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', restarted_at: null, deployed_at: null, provisioning_id: null });
|
||||
const history = betaRevisionHistory(x.betaId);
|
||||
expect(history[0]).toMatchObject({ revision: 2, notes: 'Small fix', results: null });
|
||||
expect(history[1].results?.test?.servers[0].passed).toBe(true);
|
||||
expect(history[1].results?.test?.evidence).toHaveLength(1);
|
||||
expect(x.db.prepare('SELECT content FROM beta_revision_files WHERE beta_id = ? AND revision = 1').get(x.betaId))
|
||||
.toEqual({ content: Buffer.from('revision one') });
|
||||
expect(await readFile(x.path, 'utf8')).toBe('revision one');
|
||||
await expect(x.promoteBeta(x.betaId)).rejects.toThrow('has not passed');
|
||||
const nextRun = x.db.prepare('SELECT revision, metrics FROM beta_test_runs WHERE retired_at IS NULL').get() as { revision: number; metrics: string };
|
||||
expect(nextRun.revision).toBe(2); expect(JSON.parse(nextRun.metrics).manual).toEqual({});
|
||||
});
|
||||
|
||||
it.each(['original bytes', undefined])('restores the pre-beta state after multiple revisions (original=%s)', async original => {
|
||||
const x = await setup({ original });
|
||||
await activateFiles('beta', x.betaId);
|
||||
for (let revision = 2; revision <= 3; revision++) {
|
||||
await x.reviseBeta(x.betaId, revision - 1, [{ scriptId: x.script.id, content: Buffer.from(`revision ${revision}`) }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
}
|
||||
expect(await readFile(x.path, 'utf8')).toBe('revision 3');
|
||||
expect(x.db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get())
|
||||
.toEqual({ had_original: original === undefined ? 0 : 1, prior_version_id: null });
|
||||
const result = await x.rollbackBeta(x.betaId);
|
||||
await activateFiles('campaign', result.deploymentId!);
|
||||
if (original === undefined) expect(existsSync(x.path)).toBe(false);
|
||||
else expect(await readFile(x.path, 'utf8')).toBe(original);
|
||||
});
|
||||
|
||||
it.each([false, true])('rolls back a prior activation while its replacement is still queued (waves=%s)', async waves => {
|
||||
const x = await setup({ original: 'before beta', waveSize: waves ? 1 : 0 });
|
||||
await activateFiles('beta', x.betaId);
|
||||
if (waves) {
|
||||
await x.deployBetaServers(x.betaId, [names[1]]); await activateFiles('beta', x.betaId);
|
||||
}
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('pending replacement') }]);
|
||||
if (waves) expect(x.listBetaServers(x.betaId).map(row => row.stage)).toEqual(['awaiting-empty', 'queued']);
|
||||
const result = await x.rollbackBeta(x.betaId);
|
||||
expect(result.restored).toContain(names[0]);
|
||||
if (waves) expect(result.dropped).toContain(names[1]);
|
||||
await activateFiles('campaign', result.deploymentId!);
|
||||
expect(await readFile(x.path, 'utf8')).toBe('before beta');
|
||||
});
|
||||
|
||||
it('does not restore or delete files when no revision was ever activated', async () => {
|
||||
const x = await setup({ original: 'untouched' });
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('new') }]);
|
||||
expect(await x.rollbackBeta(x.betaId)).toMatchObject({ deploymentId: null });
|
||||
expect(await readFile(x.path, 'utf8')).toBe('untouched');
|
||||
});
|
||||
|
||||
it('preserves the original tracked inventory version across revisions', async () => {
|
||||
const x = await setup({ original: 'tracked original' });
|
||||
const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const prior = x.uploadNewVersion(x.script.id, Buffer.from('tracked original'));
|
||||
recordScript(names[0], 'revision.gsc', prior.id, prior.sha256, prior.size, 'test');
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
expect(x.db.prepare('SELECT prior_version_id FROM beta_server_scripts').get()).toEqual({ prior_version_id: prior.id });
|
||||
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
|
||||
expect(getServerScript(names[0], 'revision.gsc')?.script_version_id).toBe(prior.id);
|
||||
});
|
||||
|
||||
it('starts a revision with the same original canary selection', async () => {
|
||||
const x = await setup({ waveSize: 1 });
|
||||
await x.rollbackBeta(x.betaId);
|
||||
const { betaId } = await x.openBeta({ scripts: [{ scriptId: x.script.id, scriptVersionId: x.version.id }],
|
||||
servers: [...names].reverse(), waitForHealth: true, waitForMatch: false, testPlan: x.plan });
|
||||
x.db.prepare("UPDATE beta_test_plans SET outcome = 'paused' WHERE beta_id = ?").run(betaId);
|
||||
await x.reviseBeta(betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]);
|
||||
expect(x.listBetaServers(betaId).map(row => [row.server, row.stage]))
|
||||
.toEqual([[names[0], 'queued'], [names[1], 'awaiting-empty']]);
|
||||
});
|
||||
|
||||
it('allows updates after an interrupted attempt was superseded', async () => {
|
||||
const x = await setup();
|
||||
const { requestRestart, releaseRestartRequest } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
x.db.prepare("UPDATE server_deployments SET state = 'interrupted' WHERE id = ?").run(job.id);
|
||||
releaseRestartRequest(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
await expect(x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('replacement') }]))
|
||||
.resolves.toEqual({ revision: 2 });
|
||||
});
|
||||
|
||||
it('recomposes config revisions and preserves the original shared file', async () => {
|
||||
const x = await setup({ original: 'original config', kind: 'baseline' });
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('set sv_voice "0"\n') }]);
|
||||
await activateFiles('beta', x.betaId);
|
||||
expect(await readFile(x.path, 'utf8')).toContain('set sv_voice "0"');
|
||||
const result = await x.rollbackBeta(x.betaId); await activateFiles('campaign', result.deploymentId!);
|
||||
expect(await readFile(x.path, 'utf8')).toBe('original config');
|
||||
});
|
||||
|
||||
it.each(['stale', 'identical', 'empty', 'unbundled', 'duplicate', 'closed', 'deleted', 'interrupted', 'activating'])('rejects %s updates without saving a revision', async reason => {
|
||||
const x = await setup();
|
||||
const upload = { scriptId: x.script.id, content: Buffer.from('changed') };
|
||||
if (reason === 'identical') upload.content = Buffer.from(x.version.content);
|
||||
if (reason === 'empty') upload.content = Buffer.alloc(0);
|
||||
if (reason === 'unbundled') upload.scriptId = 999;
|
||||
if (reason === 'closed') x.db.prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(x.betaId);
|
||||
if (reason === 'deleted') x.db.prepare('UPDATE beta_servers SET server_deleted_at = ? WHERE beta_id = ?').run(new Date().toISOString(), x.betaId);
|
||||
if (reason === 'interrupted') x.db.prepare("UPDATE staged_file_changes SET state = 'applying'").run();
|
||||
if (reason === 'activating') {
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const job = requestRestart(names[0], { kind: 'beta', id: String(x.betaId) });
|
||||
x.db.prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(job.id);
|
||||
}
|
||||
await expect(x.reviseBeta(x.betaId, reason === 'stale' ? 2 : 1, reason === 'duplicate' ? [upload, upload] : [upload])).rejects.toThrow();
|
||||
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
|
||||
expect(x.listVersions(x.script.id)).toHaveLength(1);
|
||||
expect(x.listBetaScripts(x.betaId)[0].script_version_id).toBe(x.version.id);
|
||||
});
|
||||
|
||||
it('keeps the new revision through reload, and uses its deadline rather than the beta opening time', async () => {
|
||||
const x = await setup();
|
||||
x.db.prepare('UPDATE betas SET opened_at = ?').run('2000-01-01T00:00:00Z');
|
||||
x.db.prepare("UPDATE beta_test_plans SET outcome = 'inconclusive'").run();
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
const { plannedBetaTick } = await import('../../src/lib/server/pipeline/beta-plan-runner.js');
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
await plannedBetaTick(); expect(testView(x.betaId)?.outcome).toBe('active');
|
||||
const { GET } = await import('../../src/routes/api/betas/[id]/+server.js');
|
||||
const body = await (await GET({ params: { id: String(x.betaId) } })).json();
|
||||
expect(body.revision.revision).toBe(2); expect(body.revisions).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('supports legacy betas without a test plan', async () => {
|
||||
const x = await setup({ legacy: true });
|
||||
await activateFiles('beta', x.betaId);
|
||||
await x.reviseBeta(x.betaId, 1, [{ scriptId: x.script.id, content: Buffer.from('changed') }]);
|
||||
expect(x.listBetaServers(x.betaId)[0]).toMatchObject({ stage: 'awaiting-empty', match_seen_at: null, healthy_at: null });
|
||||
expect(x.db.prepare('SELECT * FROM beta_test_plans').all()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('rolls back the entire revision when a later upload fails validation', async () => {
|
||||
const x = await setup();
|
||||
const overlay = x.createScript({ name: 'Overlay', kind: 'config', content: Buffer.from('set sv_voice "1"\n'), scopeType: 'game', scopeValue: 't5' });
|
||||
x.db.prepare("INSERT INTO beta_scripts (beta_id, script_id, script_version_id, kind) VALUES (?, ?, ?, 'config')")
|
||||
.run(x.betaId, overlay.script.id, overlay.version.id);
|
||||
await expect(x.reviseBeta(x.betaId, 1, [
|
||||
{ scriptId: x.script.id, content: Buffer.from('valid new version') },
|
||||
{ scriptId: overlay.script.id, content: Buffer.from('invalid config') }
|
||||
])).rejects.toThrow('at least one');
|
||||
expect(x.listVersions(x.script.id)).toHaveLength(1);
|
||||
expect(x.listVersions(overlay.script.id)).toHaveLength(1);
|
||||
expect(x.db.prepare('SELECT * FROM beta_revisions').all()).toEqual([]);
|
||||
expect(x.listBetaScripts(x.betaId).find(file => file.script_id === x.script.id)?.script_version_id).toBe(x.version.id);
|
||||
});
|
||||
|
||||
it('accepts reviewed multipart updates and downloads immutable revision files', async () => {
|
||||
const x = await setup();
|
||||
const { POST } = await import('../../src/routes/api/betas/[id]/revisions/+server.js');
|
||||
const form = new FormData(); form.set('expectedRevision', '1'); form.set('notes', 'Fix scoring');
|
||||
form.set(`file:${x.script.id}`, new File(['changed'], 'revision.gsc'));
|
||||
const post = () => POST({ params: { id: String(x.betaId) }, request: new Request('http://test/revisions', { method: 'POST', body: form }) });
|
||||
expect((await post()).status).toBe(200); expect((await post()).status).toBe(409);
|
||||
const { GET } = await import('../../src/routes/api/betas/[id]/revisions/[revision]/files/[scriptId]/+server.js');
|
||||
const download = await GET({ params: { id: String(x.betaId), revision: '1', scriptId: String(x.script.id) } });
|
||||
expect(await download.text()).toBe('revision one');
|
||||
});
|
||||
});
|
||||
@@ -1276,6 +1276,41 @@ describe('native provisioning cancellation and rollback', () => {
|
||||
});
|
||||
|
||||
describe('cancelled native mutation bookkeeping', () => {
|
||||
it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta, reviseBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { reportHostOperation, getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { script, version } = createScript({ name: 'Revised beta', kind: 'script', targetRelPath: 'revision.gsc',
|
||||
content: Buffer.from('revision one'), scopeType: 'game', scopeValue: 't5' });
|
||||
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
|
||||
servers: [job.name], waitForHealth: true, waitForMatch: false });
|
||||
async function activate(before: string | null) {
|
||||
const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
|
||||
await advance();
|
||||
const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?').get(restart.id) as { stage_operation_id: string };
|
||||
const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ id: number; path: string }>;
|
||||
reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
|
||||
await advance();
|
||||
// A dispatched activation must settle before revision submission.
|
||||
await expect(reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('premature update') }])).rejects.toThrow();
|
||||
const activation = (await operationFor(restart.id))!;
|
||||
reportHostOperation(hostId, activation.id, { state: 'completed', result: {
|
||||
before: files.map(file => ({ ...file, contentBase64: before === null ? null : Buffer.from(before).toString('base64') }))
|
||||
} });
|
||||
await advance();
|
||||
expect((await detail(restart.id)).state).toBe('done');
|
||||
}
|
||||
await activate(original);
|
||||
await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]);
|
||||
await activate('revision one');
|
||||
const rollback = await rollbackBeta(betaId);
|
||||
expect(db().prepare("SELECT content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
|
||||
.toEqual({ content: original === null ? null : Buffer.from(original) });
|
||||
});
|
||||
|
||||
it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
|
||||
const job = await queue();
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
|
||||
Reference in New Issue
Block a user