josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Verify Plutonium revisions during provisioning

Browse Source
This commit is contained in:
josh
2026-09-28 14:50:06 -04:00
parent 3fb59cd598
commit a4a5298408
19 changed files with 414 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/provisioning-detail.ts
+3 -3
View File
@@ -3,13 +3,13 @@ import type { LaunchRecipeSummary } from './deployment-plan.js';
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', 'create-deployment',
'container-action', 'remove-container', 'delete-deployment', 'cleanup', 'verify-container', 'verify-game',
'verify-map', 'verify-stopped'] as const;
'verify-map', 'verify-plutonium', 'verify-stopped'] as const;
export type ProvisioningPhase = typeof provisioningPhases[number];
export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown';
export type EvidenceSource = 'komodo' | 'rcon' | 'telemetry' | 'combined' | null;
export type EvidenceSource = 'komodo' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null;
export interface ProvisioningCheck {
key: 'container' | 'game' | 'map' | 'occupancy' | 'stopped';
key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped';
label: string;
result: EvidenceResult;
source: EvidenceSource;
src/lib/server/integrations/plutonium.ts
+31 -8
View File
@@ -35,20 +35,30 @@ export function runningVersions(): PlutoniumVersion[] {
});
}
export async function probeVersion(server: string): Promise<number | null> {
const original = getServer(server);
if (!original) return null;
const epoch = latestGameSubmission(server);
const recovering = restartRecoveryPending(server);
const health = !recovering && getServer(server)?.desired_state !== 'stopped'
const health = !recovering && original.desired_state !== 'stopped'
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
if (!getServer(server)) return null;
if (epoch !== latestGameSubmission(server) || restartRecoveryPending(server)) {
const current = getServer(server);
// An older probe must not clear evidence already recorded for a new process
// or replacement registration, including replacements with no submission yet.
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
const checkedAt = nowIso();
if (restartRecoveryPending(server)) {
result = { revision: null, error: 'Waiting for restart recovery.' };
} else if (current.desired_state === 'stopped') {
result = { revision: null, error: 'Server is intentionally stopped.' };
} else if (epoch && checkedAt <= epoch) {
result = { revision: null, error: 'Waiting for a version observation after startup.' };
}
// Clear the old revision on failure: it is no longer evidence of what is running.
db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
.run(server, result.revision, nowIso(), result.error);
.run(server, result.revision, checkedAt, result.error);
return result.revision;
}
@@ -59,6 +69,23 @@ export function checkPlutonium(): Promise<void> {
return checking;
}
async function check(): Promise<void> {
await checkRelease();
const servers = listServers();
for (let i = 0; i < servers.length; i += 4) {
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
}
writeSetting('plutonium_versions_checked_at', nowIso());
}
let checkingRelease: Promise<void> | null = null;
/** Refresh the published target without probing unrelated fleet servers. */
export function checkRelease(): Promise<void> {
if (checkingRelease) return checkingRelease;
checkingRelease = fetchRelease().finally(() => { checkingRelease = null; });
return checkingRelease;
}
async function fetchRelease(): Promise<void> {
const previous = release();
try {
const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
@@ -69,8 +96,4 @@ async function check(): Promise<void> {
} catch (err) {
writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
}
const servers = listServers();
for (let i = 0; i < servers.length; i += 4) {
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
}
}
src/lib/server/maintenance/runtime.ts
+5 -1
View File
@@ -98,7 +98,11 @@ export function maintenanceTick(): Promise<void> {
}
async function runTick(): Promise<void> {
const latest = release();
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium();
// Provisioning refreshes the manifest independently; that must not postpone
// the hourly version checks for the rest of the fleet.
const versionsCheckedAt = readSetting<string | null>('plutonium_versions_checked_at', null);
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR
|| !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR) await checkPlutonium();
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
db().transaction(() => {
const plan = schedule();
src/lib/server/provisioning/instances.ts
+46 -5
View File
@@ -27,6 +27,7 @@ import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
export type CreateServerFailure =
| 'bad-request'
@@ -871,6 +872,7 @@ async function runCreations(): Promise<CreationOutcomes> {
continue;
}
const checks: NonNullable<ProgressUpdate['checks']> = [];
let verifyingPlutonium = false;
try {
const komodo = await komodoInspectHealthy(job.server);
if (!stillCheckingCreation(id)) continue;
@@ -893,21 +895,60 @@ async function runCreations(): Promise<CreationOutcomes> {
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
continue;
}
verifyingPlutonium = true;
recordProgress(id, { phase: 'verify-plutonium', checks });
// Refresh only the release, then probe this instance. Fleet-wide checks
// can run independently and must not be required to complete a launch.
await checkRelease();
if (!stillCheckingCreation(id)) continue;
const revision = await probeVersion(job.server);
if (!stillCheckingCreation(id)) continue;
const target = freshRelease();
const published = release();
const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as
{ checked_at: string; error: string | null } | undefined;
checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest',
result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target,
message: target === null ? published.error ?? 'The production revision could not be verified.' : null });
checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon',
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass',
checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error });
if (target === null || revision === null) {
const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable')
: 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable');
deploymentStage(id, message, { phase: 'verify-plutonium', checks,
blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } });
continue;
}
if (revision < target) {
const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`;
db().transaction(() => {
deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null });
finishDeployment(id, { ok: false, message });
})();
continue;
}
db().transaction(() => {
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
phase: 'verify-map', checks, blocker: null, checkpoints: [
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry')
+ `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, {
phase: 'verify-plutonium', checks, blocker: null, checkpoints: [
{ key: 'container-action', label: 'Container startup verified' },
{ key: 'verify-container', label: 'Container readiness verified' },
{ key: 'verify-game', label: 'Game status verified' },
{ key: 'verify-map', label: 'Selected engine map verified' }
{ key: 'verify-map', label: 'Selected engine map verified' },
{ key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` }
]
});
finishDeployment(id, { ok: true });
})();
} catch (err) {
if (!stillCheckingCreation(id)) continue;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container';
const remaining = checks.filter(check => check.key !== key);
remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness',
source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, {
phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining,
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
}
}
src/lib/server/provisioning/progress.ts
+2 -2
View File
@@ -34,9 +34,9 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string'
|| typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href))
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'occupancy', 'stopped'].includes(String(c.key))
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined'].includes(String(c.source)))
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
src/lib/ui/components/ProvisioningProgress.svelte
+1 -1
View File
@@ -9,7 +9,7 @@
const progress = $derived(detail.progress);
let expanded = $state<Record<string, boolean>>({});
const evidenceLabels = { pass: 'Passed', waiting: 'Waiting', fail: 'Failed', unknown: 'Unknown' };
const sources = { komodo: 'Komodo', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry' };
const sources = { komodo: 'Komodo', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' };
</script>
<section class="execution" aria-labelledby="execution-title">
src/lib/ui/provisioning-presentation.ts
+6 -4
View File
@@ -11,13 +11,14 @@ const phaseLabels: Record<ProvisioningPhase, string> = {
prepare: 'Prepare server files', 'create-deployment': 'Create Komodo deployment', 'container-action': 'Start container',
'remove-container': 'Stop and remove container', 'delete-deployment': 'Delete Komodo deployment', cleanup: 'Remove files and registration',
'verify-container': 'Confirm container readiness', 'verify-game': 'Verify game status', 'verify-map': 'Verify selected engine map',
'verify-plutonium': 'Verify Plutonium revision',
'verify-stopped': 'Confirm container is stopped'
};
type GroupDefinition = { key: string; label: string; phases: ProvisioningPhase[] };
const request: GroupDefinition = { key: 'request', label: 'Request and eligibility', phases: ['requested', 'eligibility'] };
const creation: GroupDefinition = { key: 'create', label: 'Create and start', phases: ['startup-queue', 'prepare', 'create-deployment', 'container-action'] };
const removal: GroupDefinition = { key: 'remove', label: 'Remove the instance', phases: ['remove-container', 'delete-deployment', 'cleanup'] };
const readiness: GroupDefinition = { key: 'verify', label: 'Verify readiness', phases: ['verify-container', 'verify-game', 'verify-map'] };
const readiness: GroupDefinition = { key: 'verify', label: 'Verify readiness', phases: ['verify-container', 'verify-game', 'verify-map', 'verify-plutonium'] };
function definitions(action: ProvisioningAction): GroupDefinition[] {
if (action === 'create') return [{ ...request, phases: ['requested'] }, creation, readiness];
if (action === 'rebuild') return [request, removal, { ...creation, label: 'Create a fresh instance' }, readiness];
@@ -36,7 +37,8 @@ export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[]
return groups.map(group => {
const steps = group.phases.map((key): ProgressStep => {
const checkpoint = progress.checkpoints.find(item => item.key === key);
const checkKey = key === 'verify-container' ? 'container' : key === 'verify-game' ? 'game' : key === 'verify-map' ? 'map' : key === 'verify-stopped' ? 'stopped' : null;
const checkKey = key === 'verify-container' ? 'container' : key === 'verify-game' ? 'game' : key === 'verify-map' ? 'map'
: key === 'verify-plutonium' ? 'plutonium-version' : key === 'verify-stopped' ? 'stopped' : null;
const check = checkKey ? progress.checks.find(item => item.key === checkKey) : null;
const index = phases.indexOf(key);
const isCurrent = key === progress.phase;
@@ -185,9 +187,9 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
&& typeof progress.blocker.automatic === 'boolean' && optionalLink(progress.blocker.href)
&& (progress.blocker.label === undefined || nullableText(progress.blocker.label)))) return false;
if (!Array.isArray(progress.checks) || !progress.checks.every(check => record(check)
&& ['container', 'game', 'map', 'occupancy', 'stopped'].includes(check.key as string)
&& ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(check.key as string)
&& ['pass', 'waiting', 'fail', 'unknown'].includes(check.result as string)
&& [null, 'komodo', 'rcon', 'telemetry', 'combined'].includes(check.source as null | string)
&& [null, 'komodo', 'rcon', 'telemetry', 'combined', 'manifest'].includes(check.source as null | string)
&& text(check.label) && timestamp(check.checkedAt) && nullableTime(check.receivedAt)
&& scalar(check.expected) && scalar(check.actual) && nullableText(check.message))) return false;
if (!Array.isArray(progress.checkpoints) || !progress.checkpoints.every(checkpoint => record(checkpoint)