Limit status checks to two per host
This commit is contained in:
@@ -20,9 +20,8 @@ import { trackActivity } from './activity.js';
|
||||
import { withOperationSource } from '../hosts/operation-context.js';
|
||||
|
||||
const TICK_MS = 15_000;
|
||||
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
|
||||
// host all at once.
|
||||
const STATUS_CONCURRENCY = 4;
|
||||
// Match each agent's two health lanes without making unrelated hosts wait.
|
||||
const STATUS_CONCURRENCY_PER_HOST = 2;
|
||||
|
||||
let started = false;
|
||||
let tickTimer: NodeJS.Timeout | null = null;
|
||||
@@ -76,8 +75,8 @@ function encodeError(reason: RconFailure, detail: string): string {
|
||||
// Deduplicate queued/running checks per server. A slow server must not prevent
|
||||
// completed servers from being polled again on the next 30-second tick.
|
||||
const pendingStatus = new Map<string, Promise<number | null>>();
|
||||
const statusQueue: Array<() => Promise<void>> = [];
|
||||
let activeStatus = 0;
|
||||
interface StatusQueue { active: number; queued: Array<() => Promise<void>> }
|
||||
const statusQueues = new Map<string, StatusQueue>();
|
||||
const changedStatus = new Map<string, ServerStatusEntry>();
|
||||
function publishStatus(entry: ServerStatusEntry) {
|
||||
const scheduled = changedStatus.size > 0;
|
||||
@@ -88,21 +87,27 @@ function publishStatus(entry: ServerStatusEntry) {
|
||||
});
|
||||
}
|
||||
|
||||
function drainStatusQueue(): void {
|
||||
while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
|
||||
const next = statusQueue.shift()!;
|
||||
activeStatus++;
|
||||
void next().finally(() => { activeStatus--; drainStatusQueue(); });
|
||||
function drainStatusQueue(host: string, queue: StatusQueue): void {
|
||||
while (queue.active < STATUS_CONCURRENCY_PER_HOST && queue.queued.length) {
|
||||
const next = queue.queued.shift()!;
|
||||
queue.active++;
|
||||
void next().finally(() => { queue.active--; drainStatusQueue(host, queue); });
|
||||
}
|
||||
if (!queue.active && !queue.queued.length) statusQueues.delete(host);
|
||||
}
|
||||
|
||||
function queueStatus(server: string): Promise<number | null> {
|
||||
function queueStatus(srv: ServerRow): Promise<number | null> {
|
||||
const server = srv.name;
|
||||
const pending = pendingStatus.get(server);
|
||||
if (pending) return pending;
|
||||
// Legacy servers without a recorded host share the configured host's pool.
|
||||
const host = srv.host_id ? `native:${srv.host_id}` : `komodo:${srv.komodo_server || config.komodo.targetServer}`;
|
||||
let queue = statusQueues.get(host);
|
||||
if (!queue) { queue = { active: 0, queued: [] }; statusQueues.set(host, queue); }
|
||||
let finish!: (capacity: number | null) => void;
|
||||
const work = new Promise<number | null>(resolve => { finish = resolve; });
|
||||
pendingStatus.set(server, work);
|
||||
statusQueue.push(async () => {
|
||||
queue.queued.push(async () => {
|
||||
let capacity: number | null = null;
|
||||
try {
|
||||
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
|
||||
@@ -112,14 +117,14 @@ function queueStatus(server: string): Promise<number | null> {
|
||||
} catch (err) { console.error(`[worker] status error for ${server}:`, err); }
|
||||
finally { pendingStatus.delete(server); finish(capacity); }
|
||||
});
|
||||
drainStatusQueue();
|
||||
drainStatusQueue(host, queue);
|
||||
return work;
|
||||
}
|
||||
|
||||
export async function fanoutStatus(): Promise<void> {
|
||||
const servers = listServers();
|
||||
if (!servers.length) return;
|
||||
const results = await Promise.all(servers.map(server => queueStatus(server.name)));
|
||||
const results = await Promise.all(servers.map(queueStatus));
|
||||
const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
|
||||
try {
|
||||
// Metrics remain complete fleet collections, using freshness at capture
|
||||
|
||||
Reference in New Issue
Block a user