Preserve verified health when provisioning completes

This commit is contained in:
2026-10-06 15:48:38 -04:00
parent 6999a33aef
commit 57d2bd11db
9 changed files with 256 additions and 41 deletions
+13 -3
View File
@@ -23,6 +23,8 @@ import { trackActivity } from '../pipeline/activity.js';
import { fleetSettings } from '../settings.js';
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
import { monitoringIdentity } from '../fleet/health-observations.js';
import { handoffVerifiedHealth } from './health-handoff.js';
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
import { recordScript, removeScript } from '../fleet/inventory.js';
import { sha256Buffer } from '../fleet/files.js';
@@ -544,7 +546,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
};
deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
const verificationIdentity = monitoringIdentity(job.server);
const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
const containerObservedAt = new Date().toISOString();
if (!stillActive(job)) return;
if (job.action === 'stop') {
const stopped = ['stopped','exited','created','dead'].includes(health.state);
@@ -584,9 +588,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
}
db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
recordProgress(job.id, { blocker: null, checks });
complete(job); return;
const completed = db().transaction(() => {
if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity,
{ state: health.state, observedAt: containerObservedAt }, status)) return false;
recordProgress(job.id, { blocker: null, checks });
complete(job);
return true;
})();
if (!completed) hold(job, 'Waiting for current health evidence before completing verification.');
return;
}
const reason = eligibility(job);
if (reason) { hold(job, reason); return; }