josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Verify Plutonium revisions during provisioning

Browse Source
This commit is contained in:
josh
2026-09-28 14:50:06 -04:00
parent 3fb59cd598
commit a4a5298408
19 changed files with 414 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+46 -5
View File
@@ -27,6 +27,7 @@ import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
export type CreateServerFailure =
| 'bad-request'
@@ -871,6 +872,7 @@ async function runCreations(): Promise<CreationOutcomes> {
continue;
}
const checks: NonNullable<ProgressUpdate['checks']> = [];
let verifyingPlutonium = false;
try {
const komodo = await komodoInspectHealthy(job.server);
if (!stillCheckingCreation(id)) continue;
@@ -893,21 +895,60 @@ async function runCreations(): Promise<CreationOutcomes> {
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
continue;
}
verifyingPlutonium = true;
recordProgress(id, { phase: 'verify-plutonium', checks });
// Refresh only the release, then probe this instance. Fleet-wide checks
// can run independently and must not be required to complete a launch.
await checkRelease();
if (!stillCheckingCreation(id)) continue;
const revision = await probeVersion(job.server);
if (!stillCheckingCreation(id)) continue;
const target = freshRelease();
const published = release();
const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as
{ checked_at: string; error: string | null } | undefined;
checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest',
result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target,
message: target === null ? published.error ?? 'The production revision could not be verified.' : null });
checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon',
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass',
checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error });
if (target === null || revision === null) {
const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable')
: 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable');
deploymentStage(id, message, { phase: 'verify-plutonium', checks,
blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } });
continue;
}
if (revision < target) {
const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`;
db().transaction(() => {
deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null });
finishDeployment(id, { ok: false, message });
})();
continue;
}
db().transaction(() => {
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
phase: 'verify-map', checks, blocker: null, checkpoints: [
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry')
+ `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, {
phase: 'verify-plutonium', checks, blocker: null, checkpoints: [
{ key: 'container-action', label: 'Container startup verified' },
{ key: 'verify-container', label: 'Container readiness verified' },
{ key: 'verify-game', label: 'Game status verified' },
{ key: 'verify-map', label: 'Selected engine map verified' }
{ key: 'verify-map', label: 'Selected engine map verified' },
{ key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` }
]
});
finishDeployment(id, { ok: true });
})();
} catch (err) {
if (!stillCheckingCreation(id)) continue;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container';
const remaining = checks.filter(check => check.key !== key);
remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness',
source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, {
phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining,
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
}
}
src/lib/server/provisioning/progress.ts
+2 -2
View File
@@ -34,9 +34,9 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string'
|| typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href))
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'occupancy', 'stopped'].includes(String(c.key))
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined'].includes(String(c.source)))
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;