josh
fb48f64de1
Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely. Fixes #28 Validation: 805 tests passed, Svelte checks clean, production build passed.
82 lines
6.1 KiB
TypeScript
82 lines
6.1 KiB
TypeScript
import { json } from '@sveltejs/kit';
|
|
import { releaseRestartRequest } from '$lib/server/provisioning/index.js';
|
|
import { db } from '$lib/server/db/index.js';
|
|
import { deployBetaServers, getBeta } from '$lib/server/fleet/beta.js';
|
|
import { stagedFileChanges } from '$lib/server/fleet/staged-files.js';
|
|
import { activity, currentRun, evidence, readMetrics, saveMetrics, setOutcome, storedPlan } from '$lib/server/fleet/beta-tests.js';
|
|
import { betaState, plannedBetaTick } from '$lib/server/pipeline/beta-plan-runner.js';
|
|
import { parseBetaPlan } from '$lib/beta-plan.js';
|
|
import { kick } from '$lib/server/pipeline/worker.js';
|
|
|
|
export async function POST({ params, request }: { params: { id: string }; request: Request }) {
|
|
const id = Number(params.id);
|
|
if (!Number.isSafeInteger(id) || id < 1) return json({ error: 'Invalid beta ID' }, { status: 400 });
|
|
await plannedBetaTick();
|
|
const beta = getBeta(id);
|
|
const stored = storedPlan(id);
|
|
if (!beta || !stored) return json({ error: 'Test plan not found' }, { status: 404 });
|
|
if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 });
|
|
try {
|
|
const body = await request.json();
|
|
if (db().prepare('SELECT 1 FROM beta_servers WHERE beta_id = ? AND server_deleted_at IS NOT NULL').get(id)) {
|
|
return json({ ok: false, error: 'A beta server was deleted. Roll back this beta and open a new one to collect fresh evidence.' }, { status: 409 });
|
|
}
|
|
const plan = parseBetaPlan(JSON.parse(stored.plan));
|
|
if (body.action === 'sign-off') {
|
|
if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off');
|
|
const requirement = plan.requirements.find(r => r.id === body.requirementId && r.kind === 'manual');
|
|
const run = typeof body.server === 'string' ? currentRun(id, body.server) : undefined;
|
|
if (!requirement || !run?.observing_at) throw new Error('Checklist item or active server run not found');
|
|
if (typeof body.note !== 'string' || !body.note.trim() || body.note.length > 1000) throw new Error('Add a tester name and observation (up to 1000 characters)');
|
|
const metrics = readMetrics(run);
|
|
metrics.manual[requirement.id] = body.note.trim();
|
|
saveMetrics(run, metrics);
|
|
evidence(run, `manual:${requirement.id}:${Date.now()}`, 'manual-sign-off', { requirement: requirement.label, note: body.note.trim() });
|
|
activity(id, `${body.server}: ${requirement.label} signed off: ${body.note.trim()}`);
|
|
} else if (body.action === 'pause') {
|
|
setOutcome(id, 'paused', 'Paused by operator; deployment and evidence evaluation held');
|
|
betaState(id, false);
|
|
} else if (body.action === 'resume') {
|
|
if (stored.outcome !== 'paused') throw new Error('Only paused plans can resume; failed or expired plans need a fresh attempt');
|
|
const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL').all(id) as Parameters<typeof readMetrics>[0][];
|
|
for (const run of runs) {
|
|
const metrics = readMetrics(run);
|
|
delete metrics.match;
|
|
delete metrics.lastSample;
|
|
delete metrics.wasHealthy;
|
|
metrics.streaks = {};
|
|
for (const r of plan.requirements) if (r.kind === 'healthy-minutes') metrics.values[r.id] = 0;
|
|
saveMetrics(run, metrics);
|
|
}
|
|
setOutcome(id, 'active', null);
|
|
} else if (body.action === 'retry') {
|
|
// Fresh attempts keep their predecessors as audit history. Preserve the original deadline:
|
|
// operators open another beta if the overall testing window has expired.
|
|
if (stored.outcome === 'inconclusive') throw new Error('The testing window ended. Roll back and open a new beta with a longer deadline');
|
|
if (stagedFileChanges('beta', String(id)).some(change => change.state === 'applying')) {
|
|
throw new Error('A file write was interrupted. Inspect the server and roll back before opening a fresh beta');
|
|
}
|
|
const prepare = db().prepare("SELECT server FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL AND restarted_at IS NULL")
|
|
.all(id) as { server: string }[];
|
|
if (prepare.length) {
|
|
await deployBetaServers(id, prepare.map(row => row.server));
|
|
const failed = db().prepare("SELECT error FROM beta_servers WHERE beta_id = ? AND stage = 'failed' AND deployed_at IS NULL").get(id) as { error: string | null } | undefined;
|
|
if (failed) throw new Error(failed.error ?? 'Could not prepare beta files');
|
|
}
|
|
db().transaction(() => {
|
|
const servers = db().prepare('SELECT server FROM beta_servers WHERE beta_id = ?').all(id) as { server: string }[];
|
|
for (const { server } of servers) releaseRestartRequest(server, { kind: 'beta', id: String(id) });
|
|
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND retired_at IS NULL').run(new Date().toISOString(), id);
|
|
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', error = NULL, restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error NOT LIKE 'deploy:%'").run(id);
|
|
// SQL NULL does not satisfy NOT LIKE.
|
|
db().prepare("UPDATE beta_servers SET stage = 'awaiting-empty', restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND deployed_at IS NOT NULL AND error IS NULL").run(id);
|
|
betaState(id, false);
|
|
setOutcome(id, 'active', null);
|
|
activity(id, 'Fresh test attempt requested; deployed servers will restart when empty');
|
|
})();
|
|
} else throw new Error('Unknown test action');
|
|
void kick();
|
|
return json({ ok: true });
|
|
} catch (error) { return json({ ok: false, error: (error as Error).message }, { status: 400 }); }
|
|
}
|