josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Complete provisioning job detail pages and lifecycle evidence

Browse Source
This commit is contained in:
josh
2026-09-28 14:17:19 -04:00
parent 5b65520bbc
commit 3fb59cd598
41 changed files with 2866 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/restarts.ts
+78 -30
View File
@@ -5,6 +5,8 @@ import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
@@ -38,21 +40,28 @@ export function restartSettling(server: string): boolean {
}
export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
return rolloutBlocker(server, operationId, action)?.message ?? null;
}
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
if (conflict) {
return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true,
href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
}
const slot = slotFor(server);
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
if (slot?.healthy_at === null && !ownHandoff) {
return `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`;
return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true,
href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' };
}
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
? `Queued to ${action}; waiting for another server to recover` : null;
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
}
export type RolloutRestartResult =
@@ -71,6 +80,7 @@ export async function restartInRollout(server: string, options: { operationId: s
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
@@ -81,12 +91,20 @@ export async function restartInRollout(server: string, options: { operationId: s
applyStagedFilesForOperation(operationId, server);
} catch (err) {
const message = `File activation failed: ${(err as Error).message}`;
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
finishDeployment(operationId, { ok: false, message });
db().transaction(() => {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
finishDeployment(operationId, { ok: false, message });
})();
return { kind: 'failed', message };
}
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
});
})();
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
@@ -94,24 +112,38 @@ export async function restartInRollout(server: string, options: { operationId: s
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
checkpoint: { key: 'container-action', label: 'Container command accepted' },
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
});
})();
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
db().transaction(() => {
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
})();
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
finishDeployment(operationId!, { ok: false, message });
db().transaction(() => {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
finishDeployment(operationId!, { ok: false, message });
})();
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
@@ -145,31 +177,47 @@ async function refresh(): Promise<void> {
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
let error: string | null = null;
let players: number | null = null;
const checks: NonNullable<ProgressUpdate['checks']> = [];
try {
const health = await komodoInspectHealthy(slot.server);
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
if (!health.ok) {
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
} else {
const status = await probeServerStatus(slot.server, slot.requested_at);
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
message: status.ok ? null : status.reason });
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
else players = status.players;
}
} catch (err) {
error = `health check: ${(err as Error).message}`;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
}
const now = new Date().toISOString();
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) continue;
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
db().transaction(() => {
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) return;
if (slot.operation_id) recordProgress(slot.operation_id, {
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
});
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
}
}
}
})();
}
}