Report stalled agent lanes and bound health lock waits

2026-10-06 12:07:34 -04:00
parent 532a10a880
commit 37d380fa45
8 changed files with 263 additions and 19 deletions
@@ -0,0 +1,45 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { hostAgentLaneState, type HostAgentLane } from '../../src/lib/hosts.js';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+const inventory = { protocolVersion:1, dockerRunning:true, directoriesWritable:true, storageAvailable:100 * 1024 ** 3,
+    runtimeImages:['image'], libraries:[{id:'library',state:'ready'}] };
+const lane: HostAgentLane = {name:'health-1',alive:true,busy:false,lastPollAt:'2026-10-06T12:00:00Z',pollAgeSeconds:5};
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+
+it('flags a stopped or stalled health worker despite a live heartbeat and recovers on fresh polls', async () => {
+    const api = await import('../../src/lib/server/hosts/index.js');
+    const host = api.createHost({name:'Worker host'});
+    api.enrollHost(api.createEnrollment(host.id).token,inventory);
+    for (const broken of [{...lane,alive:false},{...lane,busy:true,pollAgeSeconds:91}]) {
+        const updated = api.heartbeat(host.id,{...inventory,agentLanes:[broken,{...lane,name:'health-2'}]});
+        expect(updated).toMatchObject({connected:true,connectionState:'online',readiness:'degraded'});
+        expect(updated.readinessReasons[0]).toContain('health check 1 worker');
+        // Diagnostics must not prevent health polls from recovering the worker.
+        const check = api.queueHostOperation(host.id,'instance.inspect',{});
+        expect(api.pendingHostOperations(host.id,'health').map(op=>op.id)).toContain(check.id);
+    }
+    expect(api.heartbeat(host.id,{...inventory,agentLanes:[lane]}).readiness).toBe('ready');
+    // Older agents remain compatible without diagnostics.
+    expect(api.heartbeat(host.id,inventory).readiness).toBe('ready');
+});
+
+it('allows long reads and mutations while detecting stalled idle workers and missing initial polls', () => {
+    expect(hostAgentLaneState({...lane,lastPollAt:null})).toBe('starting');
+    expect(hostAgentLaneState({...lane,lastPollAt:null,pollAgeSeconds:91})).toBe('stalled');
+    expect(hostAgentLaneState({...lane,busy:true,pollAgeSeconds:91})).toBe('stalled');
+    for (const name of ['read','write'] as const) {
+        expect(hostAgentLaneState({...lane,name,busy:true,pollAgeSeconds:3600})).toBe('busy');
+        expect(hostAgentLaneState({...lane,name,busy:false,pollAgeSeconds:91})).toBe('stalled');
+        expect(hostAgentLaneState({...lane,name,alive:false,busy:true})).toBe('stopped');
+    }
+});
+
+it('sanitizes lane diagnostics without rejecting a heartbeat for one malformed sample', async () => {
+    const { validateInventory } = await import('../../src/lib/server/hosts/index.js');
+    expect(validateInventory({...inventory,agentLanes:[null,{...lane,secret:'omit'},{...lane},{...lane,name:'write',pollAgeSeconds:-1}]}).agentLanes)
+        .toEqual([lane]);
+    expect(validateInventory({...inventory,agentLanes:[{...lane,pollAgeSeconds:'stale'},{...lane,lastPollAt:'invalid'}]}).agentLanes).toEqual([]);
+});