Limit status checks to two per host

This commit is contained in:
2026-10-05 15:02:19 -04:00
parent 28bd08e1ea
commit 3f5cc9d3ba
2 changed files with 79 additions and 16 deletions
+19 -14
View File
@@ -20,9 +20,8 @@ import { trackActivity } from './activity.js';
import { withOperationSource } from '../hosts/operation-context.js';
const TICK_MS = 15_000;
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
// host all at once.
const STATUS_CONCURRENCY = 4;
// Match each agent's two health lanes without making unrelated hosts wait.
const STATUS_CONCURRENCY_PER_HOST = 2;
let started = false;
let tickTimer: NodeJS.Timeout | null = null;
@@ -76,8 +75,8 @@ function encodeError(reason: RconFailure, detail: string): string {
// Deduplicate queued/running checks per server. A slow server must not prevent
// completed servers from being polled again on the next 30-second tick.
const pendingStatus = new Map<string, Promise<number | null>>();
const statusQueue: Array<() => Promise<void>> = [];
let activeStatus = 0;
interface StatusQueue { active: number; queued: Array<() => Promise<void>> }
const statusQueues = new Map<string, StatusQueue>();
const changedStatus = new Map<string, ServerStatusEntry>();
function publishStatus(entry: ServerStatusEntry) {
const scheduled = changedStatus.size > 0;
@@ -88,21 +87,27 @@ function publishStatus(entry: ServerStatusEntry) {
});
}
function drainStatusQueue(): void {
while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
const next = statusQueue.shift()!;
activeStatus++;
void next().finally(() => { activeStatus--; drainStatusQueue(); });
function drainStatusQueue(host: string, queue: StatusQueue): void {
while (queue.active < STATUS_CONCURRENCY_PER_HOST && queue.queued.length) {
const next = queue.queued.shift()!;
queue.active++;
void next().finally(() => { queue.active--; drainStatusQueue(host, queue); });
}
if (!queue.active && !queue.queued.length) statusQueues.delete(host);
}
function queueStatus(server: string): Promise<number | null> {
function queueStatus(srv: ServerRow): Promise<number | null> {
const server = srv.name;
const pending = pendingStatus.get(server);
if (pending) return pending;
// Legacy servers without a recorded host share the configured host's pool.
const host = srv.host_id ? `native:${srv.host_id}` : `komodo:${srv.komodo_server || config.komodo.targetServer}`;
let queue = statusQueues.get(host);
if (!queue) { queue = { active: 0, queued: [] }; statusQueues.set(host, queue); }
let finish!: (capacity: number | null) => void;
const work = new Promise<number | null>(resolve => { finish = resolve; });
pendingStatus.set(server, work);
statusQueue.push(async () => {
queue.queued.push(async () => {
let capacity: number | null = null;
try {
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
@@ -112,14 +117,14 @@ function queueStatus(server: string): Promise<number | null> {
} catch (err) { console.error(`[worker] status error for ${server}:`, err); }
finally { pendingStatus.delete(server); finish(capacity); }
});
drainStatusQueue();
drainStatusQueue(host, queue);
return work;
}
export async function fanoutStatus(): Promise<void> {
const servers = listServers();
if (!servers.length) return;
const results = await Promise.all(servers.map(server => queueStatus(server.name)));
const results = await Promise.all(servers.map(queueStatus));
const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
try {
// Metrics remain complete fleet collections, using freshness at capture