josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Report stalled agent lanes and bound health lock waits

Browse Source
This commit is contained in:
josh
2026-10-06 12:07:34 -04:00
parent 532a10a880
commit 37d380fa45
8 changed files with 263 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+19
View File
@@ -11,6 +11,7 @@ import { parseHostInventory } from './request-context.js';
import { fleetSettings } from '../settings.js';
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationScope, HostOperationState } from '../../hosts.js';
export const AGENT_PROTOCOL_VERSION = 1;
@@ -46,6 +47,10 @@ function hostFromRow(row: HostRow): Host {
if (!row.enrolled_at) reasons.push('Agent has not enrolled.');
else if (!row.credential_active) reasons.push('Agent credential has been revoked.');
if (!connected) reasons.push('Agent heartbeat is unavailable or stale.');
if (connected) for (const lane of inventory.agentLanes ?? []) {
const state = hostAgentLaneState(lane);
if (state === 'stopped' || state === 'stalled') reasons.push(`Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.`);
}
if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.');
if (!inventory.dockerRunning) reasons.push('Docker is not ready.');
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
@@ -287,6 +292,20 @@ export function validateInventory(value: unknown): HostInventory {
...(update.error === undefined ? {} : { error: update.error }),
...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) };
}
if (inventory.agentLanes !== undefined) {
if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4) throw new Error('Invalid agent lane inventory.');
// Keep valid lane diagnostics without discarding a host heartbeat over
// one malformed sample. Do not persist thread details or raw errors.
const names = new Set<string>();
inventory.agentLanes = inventory.agentLanes.filter(lane => {
if (!lane || !['health-1','health-2','read','write'].includes(lane.name) || names.has(lane.name)
|| typeof lane.alive !== 'boolean' || typeof lane.busy !== 'boolean'
|| typeof lane.pollAgeSeconds !== 'number' || !Number.isFinite(lane.pollAgeSeconds) || lane.pollAgeSeconds < 0
|| (lane.lastPollAt !== null && (typeof lane.lastPollAt !== 'string' || lane.lastPollAt.length > 40 || !Number.isFinite(Date.parse(lane.lastPollAt))))) return false;
names.add(lane.name);
return true;
}).map(({name,alive,busy,lastPollAt,pollAgeSeconds}) => ({name,alive,busy,lastPollAt,pollAgeSeconds}));
}
if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
!library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
return inventory;