Use database fleet registry and gate RCON polling on container readiness

This commit is contained in:
2026-09-17 17:28:38 -04:00
parent 3b1c9bf3d9
commit 885abbdffe
19 changed files with 208 additions and 41 deletions
@@ -21,6 +21,13 @@ function slotFor(server: string): RestartSlot | undefined {
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
}
/** The queued restart may still expose the previous healthy container. */
export function restartSettling(server: string): boolean {
const slot = slotFor(server);
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
export function rolloutWaitReason(server: string): string | null {
const slot = slotFor(server);
if (slot?.healthy_at === null) {
@@ -49,6 +56,7 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
+34 -4
View File
@@ -5,7 +5,9 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js';
import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js';
import { listServers, setServerField } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js';
import { scanMatchState } from '../fleet/match-log.js';
@@ -108,7 +110,15 @@ function encodeError(reason: RconFailure, detail: string): string {
return `${reason}:${detail}`;
}
async function fanoutStatus(): Promise<void> {
let statusInFlight: Promise<void> | null = null;
export function fanoutStatus(): Promise<void> {
if (statusInFlight) return statusInFlight;
statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
return statusInFlight;
}
async function refreshStatus(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
@@ -120,6 +130,28 @@ async function fanoutStatus(): Promise<void> {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch((err) => ({
ok: false as const, reason: (err as Error).message
}));
const containerState = container.ok ? 'running'
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
setServerField(srv.name, 'container_state', containerState);
if (!container.ok) {
// Container readiness precedes RCON readiness. Old replies and
// failures describe the previous process, not this startup.
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: classifyHealth(null, null, containerState),
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
reason: null, replySnippet: null, activeSince: null,
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const result = await rconStatusResult(srv.name).catch((err) => ({
ok: false as const,
endpoint: null,
@@ -231,8 +263,6 @@ export function startWorker(): void {
if (started) return;
started = true;
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
void tick();
void fanoutStatus();
tickTimer = setInterval(tick, TICK_MS);