josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native rebuild runtime verification deadlock

Browse Source
This commit is contained in:
josh
2026-10-04 14:15:05 -04:00
parent f8c60fd101
commit 1bbee98b62
4 changed files with 111 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/integrations/plutonium.ts
+10 -4
View File
@@ -35,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] {
: revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
});
}
export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise<number | null> {
export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise<number | null> {
// Startup verification calls this directly; only maintenance observers yield
// ownership, without replacing provisioning's version evidence with an error.
const suppressed = () => options.maintenance && hasActiveProvisioning(server);
const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined
&& !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server));
// The verifying operation keeps its recovery slot until every readiness
// check passes. Waiting on its own slot would deadlock native rebuilds.
const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId
&& db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId));
if (suppressed()) return null;
const original = getServer(server);
if (!original) return null;
const epoch = latestGameSubmission(server);
const recovering = restartRecoveryPending(server);
const recovering = recoveryBlocked();
const health = !recovering && original.desired_state !== 'stopped'
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
if (suppressed()) return null;
@@ -55,7 +61,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool
// or replacement registration, including replacements with no submission yet.
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
const checkedAt = nowIso();
if (restartRecoveryPending(server)) {
if (recoveryBlocked()) {
result = { revision: null, error: 'Waiting for restart recovery.' };
} else if (current.desired_state === 'stopped') {
result = { revision: null, error: 'Server is intentionally stopped.' };