josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native rebuild runtime verification deadlock

Browse Source
This commit is contained in:
josh
2026-10-04 14:15:05 -04:00
parent f8c60fd101
commit 1bbee98b62
4 changed files with 111 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/integrations/plutonium.ts
+10 -4
View File
@@ -35,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] {
: revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
});
}
export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise<number | null> {
export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise<number | null> {
// Startup verification calls this directly; only maintenance observers yield
// ownership, without replacing provisioning's version evidence with an error.
const suppressed = () => options.maintenance && hasActiveProvisioning(server);
const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined
&& !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server));
// The verifying operation keeps its recovery slot until every readiness
// check passes. Waiting on its own slot would deadlock native rebuilds.
const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId
&& db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId));
if (suppressed()) return null;
const original = getServer(server);
if (!original) return null;
const epoch = latestGameSubmission(server);
const recovering = restartRecoveryPending(server);
const recovering = recoveryBlocked();
const health = !recovering && original.desired_state !== 'stopped'
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
if (suppressed()) return null;
@@ -55,7 +61,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool
// or replacement registration, including replacements with no submission yet.
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
const checkedAt = nowIso();
if (restartRecoveryPending(server)) {
if (recoveryBlocked()) {
result = { revision: null, error: 'Waiting for restart recovery.' };
} else if (current.desired_state === 'stopped') {
result = { revision: null, error: 'Server is intentionally stopped.' };
src/lib/server/provisioning/native.ts
+1 -1
View File
@@ -319,7 +319,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const recipe = recipeForDeployment(job.id)!.recipe;
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
await checkRelease();
const revision = await probeVersion(job.server); const target = freshRelease();
const revision = await probeVersion(job.server, { operationId: job.id }); const target = freshRelease();
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
}