Fix transient health checks and avoid unnecessary recovery

tests/unit/server-status.test.ts

@@ -30,6 +30,110 @@ async function setup() {
 }
 
 describe('database fleet and container readiness', () => {
+    it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const now = Date.parse('2026-10-02T12:00:00Z');
+        vi.setSystemTime(now);
+        for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`);
+        let release!: (result: { ok: true; state: string }) => void;
+        let active = 0;
+        let peak = 0;
+        inspect.mockImplementation(async name => {
+            peak = Math.max(peak, ++active);
+            try {
+                await Promise.resolve();
+                return name === 'managed'
+                    ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
+                    : { ok: true as const, state: 'running' };
+            } finally { active--; }
+        });
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        const events: unknown[] = [];
+        const unsubscribe = subscribe(event => events.push(event));
+        const first = fanoutStatus();
+        try {
+            await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9));
+            expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([
+                expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 })
+            ]) }));
+            vi.setSystemTime(now + 30_000);
+            const second = fanoutStatus();
+            await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000));
+            expect(rcon).toHaveBeenCalledTimes(18);
+            expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1);
+            expect(peak).toBe(4);
+            release({ ok: true, state: 'running' });
+            await Promise.all([first, second]);
+            expect(rcon).toHaveBeenCalledTimes(19);
+        } finally {
+            release({ ok: true, state: 'running' });
+            await first;
+            unsubscribe();
+        }
+    });
+
+    it('expires early results before recording a fleet collection delayed by a slow check', async () => {
+        const { db, inspect, rcon, fanoutStatus } = await setup();
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const now = Date.parse('2026-10-02T12:00:00Z');
+        vi.setSystemTime(now);
+        db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run();
+        let release!: (result: { ok: true; state: string }) => void;
+        inspect.mockImplementation(async name => name === 'slow'
+            ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
+            : { ok: true as const, state: 'running' });
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        const pending = fanoutStatus();
+        try {
+            await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
+            vi.setSystemTime(now + 46_000);
+        } finally {
+            release({ ok: true, state: 'running' });
+            await pending;
+        }
+        expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([
+            { server: 'managed', players: null }, { server: 'slow', players: 2 }
+        ]);
+    });
+
+    it('keeps a manual probe diagnostic without persisting it across a restart', async () => {
+        const { db, rcon, getServer } = await setup();
+        const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
+        let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
+        rcon.mockImplementation(() => new Promise(resolve => { release = resolve; }));
+        const pending = POST({ params: { name: 'managed' } } as never);
+        db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
+        release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
+            status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        expect(await (await pending).json()).toMatchObject({ ok: true });
+        expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' });
+    });
+
+    it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        let release!: () => void;
+        const blocked = new Promise<void>(resolve => { release = resolve; });
+        inspect.mockImplementation(async () => {
+            if (phase === 'container') await blocked;
+            return { ok: true, state: 'running' };
+        });
+        rcon.mockImplementation(async () => {
+            if (phase === 'rcon') await blocked;
+            return { ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
+                status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
+        });
+        const pending = fanoutStatus();
+        try {
+            if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled());
+            db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run();
+        } finally { release(); await pending; }
+        expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null });
+    });
+
     it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => {
         const { db, inspect, rcon, fanoutStatus } = await setup();
         const { GET } = await import('../../src/routes/api/servers/+server.js');