josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Verify Plutonium revisions during provisioning

Browse Source
This commit is contained in:
josh
2026-09-28 14:50:06 -04:00
parent 3fb59cd598
commit a4a5298408
19 changed files with 414 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+46 -5
View File
@@ -27,6 +27,7 @@ import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
export type CreateServerFailure =
| 'bad-request'
@@ -871,6 +872,7 @@ async function runCreations(): Promise<CreationOutcomes> {
continue;
}
const checks: NonNullable<ProgressUpdate['checks']> = [];
let verifyingPlutonium = false;
try {
const komodo = await komodoInspectHealthy(job.server);
if (!stillCheckingCreation(id)) continue;
@@ -893,21 +895,60 @@ async function runCreations(): Promise<CreationOutcomes> {
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
continue;
}
verifyingPlutonium = true;
recordProgress(id, { phase: 'verify-plutonium', checks });
// Refresh only the release, then probe this instance. Fleet-wide checks
// can run independently and must not be required to complete a launch.
await checkRelease();
if (!stillCheckingCreation(id)) continue;
const revision = await probeVersion(job.server);
if (!stillCheckingCreation(id)) continue;
const target = freshRelease();
const published = release();
const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as
{ checked_at: string; error: string | null } | undefined;
checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest',
result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target,
message: target === null ? published.error ?? 'The production revision could not be verified.' : null });
checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon',
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass',
checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error });
if (target === null || revision === null) {
const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable')
: 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable');
deploymentStage(id, message, { phase: 'verify-plutonium', checks,
blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } });
continue;
}
if (revision < target) {
const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`;
db().transaction(() => {
deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null });
finishDeployment(id, { ok: false, message });
})();
continue;
}
db().transaction(() => {
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
phase: 'verify-map', checks, blocker: null, checkpoints: [
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry')
+ `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, {
phase: 'verify-plutonium', checks, blocker: null, checkpoints: [
{ key: 'container-action', label: 'Container startup verified' },
{ key: 'verify-container', label: 'Container readiness verified' },
{ key: 'verify-game', label: 'Game status verified' },
{ key: 'verify-map', label: 'Selected engine map verified' }
{ key: 'verify-map', label: 'Selected engine map verified' },
{ key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` }
]
});
finishDeployment(id, { ok: true });
})();
} catch (err) {
if (!stillCheckingCreation(id)) continue;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container';
const remaining = checks.filter(check => check.key !== key);
remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness',
source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, {
phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining,
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
}
}