Verify Plutonium revisions during provisioning
This commit is contained in:
@@ -35,20 +35,30 @@ export function runningVersions(): PlutoniumVersion[] {
|
||||
});
|
||||
}
|
||||
export async function probeVersion(server: string): Promise<number | null> {
|
||||
const original = getServer(server);
|
||||
if (!original) return null;
|
||||
const epoch = latestGameSubmission(server);
|
||||
const recovering = restartRecoveryPending(server);
|
||||
const health = !recovering && getServer(server)?.desired_state !== 'stopped'
|
||||
const health = !recovering && original.desired_state !== 'stopped'
|
||||
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
|
||||
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
|
||||
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
|
||||
if (!getServer(server)) return null;
|
||||
if (epoch !== latestGameSubmission(server) || restartRecoveryPending(server)) {
|
||||
const current = getServer(server);
|
||||
// An older probe must not clear evidence already recorded for a new process
|
||||
// or replacement registration, including replacements with no submission yet.
|
||||
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
|
||||
const checkedAt = nowIso();
|
||||
if (restartRecoveryPending(server)) {
|
||||
result = { revision: null, error: 'Waiting for restart recovery.' };
|
||||
} else if (current.desired_state === 'stopped') {
|
||||
result = { revision: null, error: 'Server is intentionally stopped.' };
|
||||
} else if (epoch && checkedAt <= epoch) {
|
||||
result = { revision: null, error: 'Waiting for a version observation after startup.' };
|
||||
}
|
||||
// Clear the old revision on failure: it is no longer evidence of what is running.
|
||||
db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
|
||||
.run(server, result.revision, nowIso(), result.error);
|
||||
.run(server, result.revision, checkedAt, result.error);
|
||||
return result.revision;
|
||||
}
|
||||
|
||||
@@ -59,6 +69,23 @@ export function checkPlutonium(): Promise<void> {
|
||||
return checking;
|
||||
}
|
||||
async function check(): Promise<void> {
|
||||
await checkRelease();
|
||||
const servers = listServers();
|
||||
for (let i = 0; i < servers.length; i += 4) {
|
||||
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
|
||||
}
|
||||
writeSetting('plutonium_versions_checked_at', nowIso());
|
||||
}
|
||||
|
||||
let checkingRelease: Promise<void> | null = null;
|
||||
/** Refresh the published target without probing unrelated fleet servers. */
|
||||
export function checkRelease(): Promise<void> {
|
||||
if (checkingRelease) return checkingRelease;
|
||||
checkingRelease = fetchRelease().finally(() => { checkingRelease = null; });
|
||||
return checkingRelease;
|
||||
}
|
||||
|
||||
async function fetchRelease(): Promise<void> {
|
||||
const previous = release();
|
||||
try {
|
||||
const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
|
||||
@@ -69,8 +96,4 @@ async function check(): Promise<void> {
|
||||
} catch (err) {
|
||||
writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
|
||||
}
|
||||
const servers = listServers();
|
||||
for (let i = 0; i < servers.length; i += 4) {
|
||||
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +98,11 @@ export function maintenanceTick(): Promise<void> {
|
||||
}
|
||||
async function runTick(): Promise<void> {
|
||||
const latest = release();
|
||||
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium();
|
||||
// Provisioning refreshes the manifest independently; that must not postpone
|
||||
// the hourly version checks for the rest of the fleet.
|
||||
const versionsCheckedAt = readSetting<string | null>('plutonium_versions_checked_at', null);
|
||||
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR
|
||||
|| !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR) await checkPlutonium();
|
||||
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
|
||||
db().transaction(() => {
|
||||
const plan = schedule();
|
||||
|
||||
@@ -27,6 +27,7 @@ import { recordScript } from '../fleet/inventory.js';
|
||||
import { getVersion } from '../fleet/scripts.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
|
||||
|
||||
export type CreateServerFailure =
|
||||
| 'bad-request'
|
||||
@@ -871,6 +872,7 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
continue;
|
||||
}
|
||||
const checks: NonNullable<ProgressUpdate['checks']> = [];
|
||||
let verifyingPlutonium = false;
|
||||
try {
|
||||
const komodo = await komodoInspectHealthy(job.server);
|
||||
if (!stillCheckingCreation(id)) continue;
|
||||
@@ -893,21 +895,60 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
|
||||
continue;
|
||||
}
|
||||
verifyingPlutonium = true;
|
||||
recordProgress(id, { phase: 'verify-plutonium', checks });
|
||||
// Refresh only the release, then probe this instance. Fleet-wide checks
|
||||
// can run independently and must not be required to complete a launch.
|
||||
await checkRelease();
|
||||
if (!stillCheckingCreation(id)) continue;
|
||||
const revision = await probeVersion(job.server);
|
||||
if (!stillCheckingCreation(id)) continue;
|
||||
const target = freshRelease();
|
||||
const published = release();
|
||||
const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as
|
||||
{ checked_at: string; error: string | null } | undefined;
|
||||
checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest',
|
||||
result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target,
|
||||
message: target === null ? published.error ?? 'The production revision could not be verified.' : null });
|
||||
checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon',
|
||||
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass',
|
||||
checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error });
|
||||
if (target === null || revision === null) {
|
||||
const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable')
|
||||
: 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable');
|
||||
deploymentStage(id, message, { phase: 'verify-plutonium', checks,
|
||||
blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } });
|
||||
continue;
|
||||
}
|
||||
if (revision < target) {
|
||||
const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`;
|
||||
db().transaction(() => {
|
||||
deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null });
|
||||
finishDeployment(id, { ok: false, message });
|
||||
})();
|
||||
continue;
|
||||
}
|
||||
db().transaction(() => {
|
||||
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
|
||||
phase: 'verify-map', checks, blocker: null, checkpoints: [
|
||||
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry')
|
||||
+ `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, {
|
||||
phase: 'verify-plutonium', checks, blocker: null, checkpoints: [
|
||||
{ key: 'container-action', label: 'Container startup verified' },
|
||||
{ key: 'verify-container', label: 'Container readiness verified' },
|
||||
{ key: 'verify-game', label: 'Game status verified' },
|
||||
{ key: 'verify-map', label: 'Selected engine map verified' }
|
||||
{ key: 'verify-map', label: 'Selected engine map verified' },
|
||||
{ key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` }
|
||||
]
|
||||
});
|
||||
finishDeployment(id, { ok: true });
|
||||
})();
|
||||
} catch (err) {
|
||||
if (!stillCheckingCreation(id)) continue;
|
||||
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
|
||||
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
|
||||
const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container';
|
||||
const remaining = checks.filter(check => check.key !== key);
|
||||
remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness',
|
||||
source: null, result: 'unknown', message: (err as Error).message });
|
||||
deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, {
|
||||
phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining,
|
||||
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,9 +34,9 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
|
||||
if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string'
|
||||
|| typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href))
|
||||
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
|
||||
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'occupancy', 'stopped'].includes(String(c.key))
|
||||
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
|
||||
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
|
||||
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined'].includes(String(c.source)))
|
||||
&& (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
|
||||
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
|
||||
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
|
||||
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
|
||||
|
||||
Reference in New Issue
Block a user