josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Replace IW4MAdmin with direct RCON for player counts

Browse Source 
IW4M kept hitting a network wall (container -> host LAN IP -> IW4M
loopback bind) that we couldn't cleanly punch through, and the two-minute
!rt token added enough operational friction that the pause-all-deployments
gate never got a fair test. RCON gives us the same fact — how many
players are on a server right now — direct from each game server over
UDP, with no session, no token, no proxy.

- Add integrations/rcon.ts: dgram UDP client. rconGetPlayerCount() and
  rconGetStatus() parse both `status` (print + scoreboard) and
  `statusResponse` (info line + rows) dialects. Timeout via
  RCON_TIMEOUT_MS.
- Add komodoResolveGamePort() + extractUdpPort(). Reads Komodo's read
  API for a server, walks both docker-compose-style port objects and
  docker-run-style "host:container/udp" strings, caches into
  servers.game_port on first success.
- Replace pipelineReady() guard in sweep with a simple
  integrations-configured check. Same held/failed/manual outcomes,
  but keyed on RCON+Komodo instead of IW4M+auth-state.
- Worker's status refresh fans out per-server over RCON at concurrency=4
  so a 15-server fleet doesn't spike UDP.
- Schema: drop iw4m_match/iw4m_value, add game_port + rcon_host.
- Delete every auth artifact: TokenModal, auth store, auth-state module,
  /api/auth/token, /api/events/auth, iw4m.test.ts. Layout no longer
  mounts the modal, pipeline store no longer tracks a frozen state.
- README + .env.example + docker-compose.yml + komodo/stack.toml swap
  IW4M_* env for RCON_PASSWORD / RCON_HOST / RCON_TIMEOUT_MS. Same
  host.docker.internal:host-gateway extra_hosts entry, repurposed
  for RCON reach.
- Tests: add rcon.test.ts and komodo.test.ts, update sweep.test.ts to
  drop the auth-freeze case. 23/23 passing.
This commit is contained in:
josh
2026-09-12 01:33:06 -04:00
parent f118c179cd
commit 854f5532a9
27 changed files with 530 additions and 730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/config.ts
+11 -5
View File
@@ -26,9 +26,15 @@ export const config = {
apiKey: env.KOMODO_API_KEY ?? '',
apiSecret: env.KOMODO_API_SECRET ?? ''
},
iw4m: {
address: env.IW4M_ADDRESS ?? '',
clientId: env.IW4M_CLIENT_ID ?? ''
rcon: {
// Host every RCON packet is sent to. Defaults to the Docker
// host-gateway alias so the electric-cherry container can reach
// Plutonium servers running on the same host without knowing each
// container's internal IP.
host: env.RCON_HOST ?? 'host.docker.internal',
// Shared across the fleet — every server uses the same rcon_password.
password: env.RCON_PASSWORD ?? '',
timeoutMs: num('RCON_TIMEOUT_MS', 3_000)
}
} as const;
@@ -36,8 +42,8 @@ export function komodoConfigured(): boolean {
return !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret);
}
export function iw4mConfigured(): boolean {
return !!(config.iw4m.address && config.iw4m.clientId);
export function rconConfigured(): boolean {
return !!config.rcon.password;
}
export const betaSrc = () => `${config.sharedScripts}/betas`;
src/lib/server/db/schema.sql
+2 -2
View File
@@ -9,8 +9,8 @@ CREATE TABLE IF NOT EXISTS servers (
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
komodo_name TEXT NOT NULL,
komodo_server TEXT,
iw4m_match TEXT, -- port|id|name
iw4m_value TEXT,
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
rcon_host TEXT, -- optional per-server override of RCON_HOST
last_players INTEGER,
last_checked_at TEXT,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
src/lib/server/fleet/servers.ts
+4 -4
View File
@@ -67,8 +67,8 @@ export interface ServerRow {
komodo_type: 'deployment' | 'stack' | 'container';
komodo_name: string;
komodo_server: string | null;
iw4m_match: 'port' | 'id' | 'name' | null;
iw4m_value: string | null;
game_port: number | null;
rcon_host: string | null;
last_players: number | null;
last_checked_at: string | null;
created_at: string;
@@ -113,8 +113,8 @@ export function setServerField(
'komodo_type',
'komodo_name',
'komodo_server',
'iw4m_match',
'iw4m_value',
'game_port',
'rcon_host',
'last_players',
'last_checked_at'
];
src/lib/server/integrations/auth-state.ts
-54
View File
@@ -1,54 +0,0 @@
import { emit } from '../pipeline/bus.js';
export type AuthState =
| { kind: 'ready'; token: string; cookie: string; expiresAt: number }
| { kind: 'stale'; reason: 'expired' | 'rejected' | 'unreachable' }
| { kind: 'missing' };
// Two minutes minus 5s slack — the !rt token really is single-use and dies
// fast. Better to preemptively mark stale than to fire a call with a dead
// session cookie mid-sweep.
export const TOKEN_TTL_MS = 115_000;
// How long before expiry the UI should re-prompt so the user has time to
// generate a new one in-game.
export const TOKEN_WARN_MS = 20_000;
let state: AuthState = { kind: 'missing' };
export function getAuthState(): AuthState {
// Auto-transition from ready → stale when we cross the expiry boundary
if (state.kind === 'ready' && Date.now() >= state.expiresAt) {
state = { kind: 'stale', reason: 'expired' };
emit({ type: 'auth-state', state });
}
return state;
}
export function setAuthState(next: AuthState): void {
state = next;
const payload: Parameters<typeof emit>[0] =
next.kind === 'ready'
? { type: 'auth-state', state: next, msRemaining: Math.max(0, next.expiresAt - Date.now()) }
: { type: 'auth-state', state: next };
emit(payload);
}
/**
* Web-app-side check used by the pipeline worker's tick loop. Returns true
* only when the token is ready AND has more than a couple seconds left. Any
* other state hard-freezes the pipeline.
*/
export function pipelineReady(): boolean {
const s = getAuthState();
if (s.kind !== 'ready') return false;
return s.expiresAt - Date.now() > 3000;
}
/**
* The UI shows a warning modal once we're within TOKEN_WARN_MS of expiry.
*/
export function tokenNearExpiry(): boolean {
const s = getAuthState();
if (s.kind !== 'ready') return false;
return s.expiresAt - Date.now() <= TOKEN_WARN_MS;
}
src/lib/server/integrations/iw4m.ts
-176
View File
@@ -1,176 +0,0 @@
import { config, iw4mConfigured } from '../config.js';
import { getServer, setServerField } from '../fleet/servers.js';
import { getAuthState, setAuthState, TOKEN_TTL_MS } from './auth-state.js';
// Ports iw4m_login/fetch_status/players/autolink from push_scripts.sh:497-620.
// The one big difference: the script keeps a temp cookie-jar file, this holds
// the session cookie in memory (we're a single Node process, not a shell).
export interface StatusEntry {
id?: number;
port?: number;
name?: string;
currentPlayers?: number;
}
let statusCache: { entries: StatusEntry[]; at: number } | null = null;
const STATUS_TTL_MS = 5_000;
function abortableFetch(url: string, init: RequestInit = {}): Promise<Response> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), config.httpTimeoutMs);
return fetch(url, { ...init, signal: controller.signal }).finally(() => clearTimeout(timer));
}
/**
* Exchange the 4-char !rt token for a session cookie. Called by the
* `/api/auth/token` route when the user pastes the modal input.
*/
export async function loginWithToken(
token: string
): Promise<{ ok: true; cookie: string; expiresAt: number } | { ok: false; reason: 'rejected' | 'unreachable' | 'unconfigured'; message: string }> {
if (!iw4mConfigured()) return { ok: false, reason: 'unconfigured', message: 'iw4m not configured' };
const url = `${config.iw4m.address.replace(/\/+$/, '')}/api/client/${config.iw4m.clientId}/login`;
let res: Response;
try {
res = await abortableFetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ password: token })
});
} catch (err) {
return { ok: false, reason: 'unreachable', message: (err as Error).message };
}
if (res.status === 401 || res.status === 403) return { ok: false, reason: 'rejected', message: 'iw4m rejected the token' };
if (!res.ok) return { ok: false, reason: 'unreachable', message: `HTTP ${res.status}` };
// Grab every Set-Cookie the server handed us and re-send them intact.
// `getSetCookie` is Node 20+; on older Node use `res.headers.raw()['set-cookie']`.
let setCookies: string[] = [];
if (typeof (res.headers as unknown as { getSetCookie?: () => string[] }).getSetCookie === 'function') {
setCookies = (res.headers as unknown as { getSetCookie: () => string[] }).getSetCookie();
} else {
const single = res.headers.get('set-cookie');
if (single) setCookies = [single];
}
const cookie = setCookies
.map((c) => c.split(';')[0])
.filter(Boolean)
.join('; ');
if (!cookie) return { ok: false, reason: 'rejected', message: 'iw4m returned no session cookie' };
return { ok: true, cookie, expiresAt: Date.now() + TOKEN_TTL_MS };
}
/**
* Fetch /api/status, using the current session cookie if we have one. Falls
* back to anonymous read — that's what the script does at line 542 — but the
* pipeline worker gates on the token being present anyway, so this is used
* mainly for the dashboard's non-blocking status refresh.
*/
export async function fetchStatus(): Promise<{ ok: true; entries: StatusEntry[] } | { ok: false; message: string }> {
if (!iw4mConfigured()) return { ok: false, message: 'iw4m not configured' };
if (statusCache && Date.now() - statusCache.at < STATUS_TTL_MS) {
return { ok: true, entries: statusCache.entries };
}
const url = `${config.iw4m.address.replace(/\/+$/, '')}/api/status`;
const s = getAuthState();
const headers: Record<string, string> = {};
if (s.kind === 'ready') headers.cookie = s.cookie;
let res: Response;
try {
res = await abortableFetch(url, { headers });
} catch (err) {
return { ok: false, message: (err as Error).message };
}
if (!res.ok) {
if ((res.status === 401 || res.status === 403) && s.kind === 'ready') {
// Cookie went sour — force a re-prompt.
setAuthState({ kind: 'stale', reason: 'rejected' });
}
return { ok: false, message: `HTTP ${res.status}` };
}
let json: unknown;
try {
json = await res.json();
} catch (err) {
return { ok: false, message: `bad JSON: ${(err as Error).message}` };
}
if (!Array.isArray(json)) return { ok: false, message: 'status was not an array' };
// Normalize keys — the script does the same case-insensitive dance
// (line 589) to work across IW4MAdmin versions.
const entries: StatusEntry[] = json.map((e) => {
const o = e as Record<string, unknown>;
return {
id: (o.id ?? o.Id) as number | undefined,
port: (o.port ?? o.Port) as number | undefined,
name: (o.name ?? o.Name ?? '') as string,
currentPlayers: (o.currentPlayers ?? o.CurrentPlayers) as number | undefined
};
});
statusCache = { entries, at: Date.now() };
return { ok: true, entries };
}
function normalizeName(s: string): string {
return s.toLowerCase().replace(/[^a-z0-9]/g, '');
}
/**
* How many players are on this server right now, or null if we can't tell.
* Ports `iw4m_players` in push_scripts.sh:579.
*/
export async function playersOn(server: string): Promise<number | null> {
const row = getServer(server);
if (!row) return null;
const status = await fetchStatus();
if (!status.ok) return null;
let match = row.iw4m_match;
let value = row.iw4m_value;
if (!match || !value) {
const auto = autolink(server, status.entries);
if (!auto) return null;
setServerField(server, 'iw4m_match', auto.match);
setServerField(server, 'iw4m_value', auto.value);
match = auto.match;
value = auto.value;
}
const hits = status.entries.filter((e) => {
if (match === 'port') return String(e.port ?? '') === value;
if (match === 'id') return String(e.id ?? '') === value;
return normalizeName(e.name ?? '') === normalizeName(value!);
});
if (hits.length !== 1) return null;
const players = hits[0].currentPlayers;
return typeof players === 'number' ? players : null;
}
/**
* First-run guess: pick the first non-generic token in the server folder
* name and look for a matching IW4M server by name. Ports `iw4m_autolink`
* in push_scripts.sh:605.
*/
function autolink(server: string, entries: StatusEntry[]): { match: 'port'; value: string } | null {
const skip = new Set(['t4', 't5', 't6', 'iw5', 'iw4']);
const parts = server
.replace(/[^a-zA-Z0-9]+/g, ' ')
.trim()
.split(' ')
.filter((p) => p && !skip.has(p.toLowerCase()) && !/^\d+$/.test(p));
if (parts.length === 0) return null;
const token = parts[0].toLowerCase();
const hits = entries.filter((e) => normalizeName(e.name ?? '').includes(token));
if (hits.length !== 1) return null;
const port = hits[0].port;
if (typeof port !== 'number') return null;
return { match: 'port', value: String(port) };
}
src/lib/server/integrations/komodo.ts
+102
View File
@@ -79,3 +79,105 @@ export async function komodoRestart(serverName: string): Promise<KomodoRestartRe
}
return { ok: true };
}
/**
* Query Komodo's read API for a server's UDP game port. Called lazily by
* the RCON client and cached into `servers.game_port` on success.
*
* Komodo returns full deployment/container state; port mappings live under
* `state.container.ports` on deployments and `state.ports` on stacks/raw
* containers. Shape varies by Komodo version, so we walk a few known
* candidate paths and pick the first UDP entry we can parse.
*/
export async function komodoResolveGamePort(serverName: string): Promise<number | null> {
if (!komodoConfigured()) return null;
const row = getServer(serverName);
if (!row?.komodo_name) return null;
const type = row.komodo_type ?? 'deployment';
let path: string;
let body: Record<string, string>;
switch (type) {
case 'stack':
path = 'read/GetStack';
body = { stack: row.komodo_name };
break;
case 'container':
if (!row.komodo_server) return null;
path = 'read/InspectContainer';
body = { server: row.komodo_server, container: row.komodo_name };
break;
case 'deployment':
default:
path = 'read/GetDeployment';
body = { deployment: row.komodo_name };
break;
}
let res: Response;
try {
res = await komodoCall(path, body);
} catch {
return null;
}
if (!res.ok) return null;
let json: unknown;
try {
json = await res.json();
} catch {
return null;
}
return extractUdpPort(json);
}
/**
* Walks a Komodo response looking for a UDP port. Handles the two shapes
* we've seen in the wild:
*
* docker-compose-style: [{ protocol: "udp", published: 4976, target: 28960 }, ...]
* docker-run-style: ["4976:28960/udp", "3074:3074/tcp"]
*
* Returns the first published port whose protocol is UDP.
*/
export function extractUdpPort(payload: unknown): number | null {
const candidates: unknown[] = [];
const seen = new Set<unknown>();
const walk = (v: unknown): void => {
if (v === null || typeof v !== 'object' || seen.has(v)) return;
seen.add(v);
if (Array.isArray(v)) {
for (const item of v) walk(item);
candidates.push(v);
} else {
for (const val of Object.values(v as Record<string, unknown>)) walk(val);
}
};
walk(payload);
// Object-form entries: { protocol, published, target }
for (const arr of candidates) {
for (const entry of arr as unknown[]) {
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
const e = entry as Record<string, unknown>;
const proto = String(e.protocol ?? e.Protocol ?? '').toLowerCase();
if (proto !== 'udp') continue;
const raw = e.published ?? e.PublishedPort ?? e.public_port ?? e.HostPort ?? e.host_port;
const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw);
if (Number.isFinite(num) && num > 0) return num;
}
}
}
// String-form entries: "4976:28960/udp"
for (const arr of candidates) {
for (const entry of arr as unknown[]) {
if (typeof entry !== 'string') continue;
const m = /^(\d+):\d+\/udp$/i.exec(entry);
if (m) return Number.parseInt(m[1], 10);
}
}
return null;
}
src/lib/server/integrations/rcon.ts
+167
View File
@@ -0,0 +1,167 @@
import { createSocket } from 'node:dgram';
import type { Socket } from 'node:dgram';
import { config, rconConfigured } from '../config.js';
import { getServer, setServerField } from '../fleet/servers.js';
import { komodoResolveGamePort } from './komodo.js';
// Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like
// \xff\xff\xff\xffrcon <password> <command>
// Responses look like
// \xff\xff\xff\xffprint\n... (older builds)
// \xff\xff\xff\xffstatusResponse\n<info-line>\n<player rows> (getstatus)
// `status` returns a `print` response with a human-readable header table
// followed by one row per connected player. We parse either — first try
// print (from `status`) which gives us a clean player row count; fall back
// to statusResponse if the server dialect prefers that.
const PREFIX = Buffer.from([0xff, 0xff, 0xff, 0xff]);
async function sendRcon(host: string, port: number, command: string): Promise<Buffer | null> {
if (!rconConfigured()) return null;
const payload = Buffer.concat([
PREFIX,
Buffer.from(`rcon ${config.rcon.password} ${command}\n`, 'utf8')
]);
return new Promise<Buffer | null>((resolve) => {
const sock: Socket = createSocket('udp4');
let settled = false;
const finish = (val: Buffer | null): void => {
if (settled) return;
settled = true;
try {
sock.close();
} catch {
/* already closed */
}
resolve(val);
};
const timer = setTimeout(() => finish(null), config.rcon.timeoutMs);
sock.on('message', (msg) => {
clearTimeout(timer);
finish(msg);
});
sock.on('error', () => {
clearTimeout(timer);
finish(null);
});
sock.send(payload, 0, payload.length, port, host, (err) => {
if (err) {
clearTimeout(timer);
finish(null);
}
});
});
}
/**
* Resolve the RCON endpoint (host + port) for a server, filling
* `servers.game_port` from Komodo on first miss. Returns null when we
* still can't determine a port after asking Komodo.
*/
async function endpointFor(server: string): Promise<{ host: string; port: number } | null> {
const row = getServer(server);
if (!row) return null;
let port = row.game_port;
if (!port) {
port = await komodoResolveGamePort(server);
if (port) setServerField(server, 'game_port', port);
}
if (!port) return null;
return { host: row.rcon_host ?? config.rcon.host, port };
}
function stripPrefix(buf: Buffer): string | null {
if (buf.length < PREFIX.length) return null;
for (let i = 0; i < PREFIX.length; i++) if (buf[i] !== PREFIX[i]) return null;
return buf.subarray(PREFIX.length).toString('utf8');
}
/**
* Count the number of player rows in a `status` response. The `print`
* response body looks like:
*
* print
* map: mp_kino
* num score ping guid name
* --- ----- ---- -------- ----
* 0 12 45 abc123...def joe
* 1 0 72 xyz789...abc jill
*
* We look for the `--- ---` separator and count non-blank lines after it.
* Falls through to a statusResponse parse if the payload uses that dialect.
*/
export function parsePlayerCount(text: string): number | null {
const lines = text.split(/\r?\n/);
const sep = lines.findIndex((l) => /^-+\s+-+/.test(l.trim()));
if (sep !== -1) {
let n = 0;
for (let i = sep + 1; i < lines.length; i++) {
const line = lines[i].trim();
if (!line) continue;
// Player rows begin with the client num (digits) then whitespace.
if (/^\d+\s+/.test(line)) n++;
}
return n;
}
// statusResponse dialect: info line starts with '\', players start on
// the line after. Same idea — count non-blank lines past the info.
const info = lines.findIndex((l) => l.startsWith('\\'));
if (info === -1) return null;
let n = 0;
for (let i = info + 1; i < lines.length; i++) {
if (lines[i].trim()) n++;
}
return n;
}
/**
* Parse `\key\value\key\value…` from a statusResponse info line.
*/
export function parseInfoLine(text: string): Record<string, string> {
const info = text.split(/\r?\n/).find((l) => l.startsWith('\\'));
if (!info) return {};
const out: Record<string, string> = {};
const parts = info.split('\\').filter(Boolean);
for (let i = 0; i < parts.length - 1; i += 2) out[parts[i]] = parts[i + 1];
return out;
}
export async function rconGetPlayerCount(server: string): Promise<number | null> {
const ep = await endpointFor(server);
if (!ep) return null;
const buf = await sendRcon(ep.host, ep.port, 'status');
if (!buf) return null;
const text = stripPrefix(buf);
if (!text) return null;
return parsePlayerCount(text);
}
export interface RconStatus {
players: number;
map: string | null;
gametype: string | null;
}
export async function rconGetStatus(server: string): Promise<RconStatus | null> {
const ep = await endpointFor(server);
if (!ep) return null;
// `getstatus` is anonymous on most CoD builds but Plutonium requires the
// authenticated rcon route for reliable results, so we ask via rcon.
const buf = await sendRcon(ep.host, ep.port, 'getstatus');
if (!buf) return null;
const text = stripPrefix(buf);
if (!text) return null;
const info = parseInfoLine(text);
const players = parsePlayerCount(text);
if (players === null) return null;
return {
players,
map: info.mapname ?? info.sv_mapname ?? null,
gametype: info.g_gametype ?? null
};
}
src/lib/server/pipeline/bus.ts
+3 -4
View File
@@ -1,8 +1,8 @@
import { EventEmitter } from 'node:events';
// One in-process event bus, multiplexed by all three SSE endpoints. Kept
// separate from any specific subscriber so push/beta/sweep can emit without
// pulling the whole worker module in.
// One in-process event bus, multiplexed by the SSE endpoints. Kept separate
// from any specific subscriber so push/beta/sweep can emit without pulling
// the whole worker module in.
export type PipelineEvent =
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
@@ -16,7 +16,6 @@ export type PipelineEvent =
error: string | null;
}
| { type: 'sweep-tick'; frozen: boolean; reason?: string }
| { type: 'auth-state'; state: import('../integrations/auth-state.js').AuthState; msRemaining?: number }
| { type: 'server-status'; entries: Array<{ server: string; players: number | null; name?: string }> };
class Bus extends EventEmitter {
src/lib/server/pipeline/sweep.ts
+26 -32
View File
@@ -1,9 +1,9 @@
import { config, iw4mConfigured, komodoConfigured } from '../config.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { fetchStatus, playersOn } from '../integrations/iw4m.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { komodoRestart } from '../integrations/komodo.js';
import { pipelineReady } from '../integrations/auth-state.js';
import { setServerField } from '../fleet/servers.js';
interface PendingRow {
deployment_id: string;
@@ -16,24 +16,21 @@ function nowIso(): string {
}
function integrationsOn(): boolean {
return komodoConfigured() && iw4mConfigured();
return komodoConfigured() && rconConfigured();
}
/**
* Re-check every pending/failed deployment_server row, restart the ones with
* zero players, mark the busy ones held. Same-server dedup is per-sweep, so
* one restart resolves every pending row for that server. Ports `deploy_sweep`
* in push_scripts.sh:705.
* zero players, hold the busy ones. Same-server dedup is per-sweep, so one
* restart resolves every pending row for that server. Ports `deploy_sweep`
* in push_scripts.sh:705 but talks RCON directly instead of routing through
* IW4MAdmin.
*/
export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number; frozen: boolean }> {
if (!pipelineReady()) {
emit({ type: 'sweep-tick', frozen: true, reason: 'iw4m token stale' });
return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: true };
}
export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number }> {
if (!integrationsOn()) {
// Same behavior as line 714 — mark rows manual so they don't hang
// forever if integrations were never configured.
// Same shape as the shell script's line 714: mark rows manual so
// they don't hang forever waiting for something that will never
// arrive.
db()
.prepare(
`UPDATE deployment_servers
@@ -41,7 +38,8 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
WHERE restart_state IN ('pending','failed')`
)
.run(nowIso());
return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: false };
emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
}
const pending = db()
@@ -51,19 +49,9 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
WHERE restart_state IN ('pending','failed')`
)
.all() as PendingRow[];
if (pending.length === 0) return { checked: 0, restarted: 0, held: 0, failed: 0, frozen: false };
// Warm the /api/status cache so per-server calls just hit memory
const status = await fetchStatus();
if (!status.ok) {
// Same as line 742 — mark held with the iw4m error
const stmt = db().prepare(
`UPDATE deployment_servers
SET checked_at = ?, error = ?, attempts = attempts + 1
WHERE restart_state IN ('pending','failed')`
);
stmt.run(nowIso(), `iw4m: ${status.message}`);
return { checked: 0, restarted: 0, held: pending.length, failed: 0, frozen: false };
if (pending.length === 0) {
emit({ type: 'sweep-tick', frozen: false });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
}
// Dedup across deployments: if we resolved server X once this tick,
@@ -87,9 +75,9 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
let outcome = seen.get(row.server);
if (!outcome) {
checked++;
const players = await playersOn(row.server);
const players = await rconGetPlayerCount(row.server);
if (players === null) {
outcome = { state: 'pending', players: null, error: 'no iw4m match' };
outcome = { state: 'pending', players: null, error: 'rcon unreachable' };
held++;
} else if (players > 0) {
outcome = { state: 'pending', players, error: null };
@@ -108,6 +96,12 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
}
}
seen.set(row.server, outcome);
// Cache the last known player count on the server row so the
// dashboard can render a value between rcon polls.
if (outcome.players !== null) {
setServerField(row.server, 'last_players', outcome.players);
setServerField(row.server, 'last_checked_at', nowIso());
}
}
const now = nowIso();
applyOutcome.run(
@@ -159,5 +153,5 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
.run(cutoff);
emit({ type: 'sweep-tick', frozen: false });
return { checked, restarted, held, failed, frozen: false };
return { checked, restarted, held, failed };
}
src/lib/server/pipeline/worker.ts
+29 -30
View File
@@ -1,11 +1,13 @@
import { sweep } from './sweep.js';
import { emit } from './bus.js';
import { fetchStatus } from '../integrations/iw4m.js';
import { rconGetStatus } from '../integrations/rcon.js';
import { listServers, seedServerRegistry } from '../fleet/servers.js';
import { getAuthState } from '../integrations/auth-state.js';
const TICK_MS = 15_000;
const STATUS_REFRESH_MS = 30_000;
// Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the
// host all at once.
const STATUS_CONCURRENCY = 4;
let started = false;
let tickTimer: NodeJS.Timeout | null = null;
@@ -19,44 +21,41 @@ async function tick(): Promise<void> {
}
}
async function pushServerStatus(): Promise<void> {
const st = await fetchStatus();
if (!st.ok) return;
const rows = listServers();
const byName = new Map(rows.map((r) => [r.name, r]));
// For the dashboard: emit a lightweight snapshot. Actual per-server
// resolution is handled by playersOn() during a sweep, which caches
// iw4m_match/value into the servers table.
emit({
type: 'server-status',
entries: st.entries.map((e) => ({
server: e.name ?? String(e.port ?? ''),
players: e.currentPlayers ?? null,
name: e.name
}))
});
// Also fold current auth ttl into the auth channel so subscribers get a
// heartbeat and a countdown without waiting for a transition.
const auth = getAuthState();
emit({
type: 'auth-state',
state: auth,
msRemaining: auth.kind === 'ready' ? Math.max(0, auth.expiresAt - Date.now()) : undefined
});
void byName; // reserved: future join between /api/status and server registry
async function fanoutStatus(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
const entries: Array<{ server: string; players: number | null; name?: string }> = new Array(servers.length);
let cursor = 0;
async function pump(): Promise<void> {
while (true) {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
const status = await rconGetStatus(srv.name).catch(() => null);
entries[i] = {
server: srv.name,
players: status?.players ?? null,
name: status?.map ?? undefined
};
}
}
await Promise.all(Array.from({ length: STATUS_CONCURRENCY }, pump));
emit({ type: 'server-status', entries });
}
export function startWorker(): void {
if (started) return;
started = true;
// Seed the servers table from the fs so first-boot has rows to reference.
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
void tick();
void pushServerStatus();
void fanoutStatus();
tickTimer = setInterval(tick, TICK_MS);
statusTimer = setInterval(pushServerStatus, STATUS_REFRESH_MS);
statusTimer = setInterval(fanoutStatus, STATUS_REFRESH_MS);
console.log('[worker] started — sweep every', TICK_MS, 'ms, status every', STATUS_REFRESH_MS, 'ms');
}