josh
e5c0fda1ed
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
516 lines
21 KiB
TypeScript
516 lines
21 KiB
TypeScript
import { createSocket } from 'node:dgram';
|
|
import type { Socket } from 'node:dgram';
|
|
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
|
import { getServer, setServerField } from '../fleet/servers.js';
|
|
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
|
|
import { komodoResolveGamePort } from './komodo.js';
|
|
import { parsePlutoniumRevision } from '../../plutonium.js';
|
|
import { nativeInstance, nativeRequest } from '../backends/native.js';
|
|
|
|
// Plutonium/CoD RCON is UDP with a 4-byte 0xff prefix. Requests look like
|
|
// \xff\xff\xff\xffrcon <password> <command>
|
|
// Responses look like
|
|
// \xff\xff\xff\xffprint\n... (older builds)
|
|
// \xff\xff\xff\xffstatusResponse\n<info-line>\n<player rows> (getstatus)
|
|
// `status` returns a `print` response with a human-readable header table
|
|
// followed by one row per connected player. We parse either — first try
|
|
// print (from `status`) which gives us a clean player row count; fall back
|
|
// to statusResponse if the server dialect prefers that.
|
|
|
|
const PREFIX = Buffer.from([0xff, 0xff, 0xff, 0xff]);
|
|
|
|
// Distinct RCON failure modes. Surfaced to the UI so the operator can tell
|
|
// "wrong password" from "wrong host" from "Plutonium isn't listening".
|
|
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
|
|
|
|
export interface RconEndpoint {
|
|
host: string;
|
|
port: number;
|
|
}
|
|
|
|
type SendResult =
|
|
| { ok: true; buf: Buffer }
|
|
| { ok: false; reason: 'no-config' | 'send-error' | 'timeout'; detail: string };
|
|
|
|
async function sendRcon(host: string, port: number, command: string): Promise<SendResult> {
|
|
if (!rconConfigured()) {
|
|
return { ok: false, reason: 'no-config', detail: 'RCON_PASSWORD is not set' };
|
|
}
|
|
const payload = Buffer.concat([
|
|
PREFIX,
|
|
Buffer.from(`rcon ${config.rcon.password} ${command}\n`, 'utf8')
|
|
]);
|
|
|
|
return new Promise<SendResult>((resolve) => {
|
|
const sock: Socket = createSocket('udp4');
|
|
let settled = false;
|
|
|
|
const finish = (val: SendResult): void => {
|
|
if (settled) return;
|
|
settled = true;
|
|
try {
|
|
sock.close();
|
|
} catch {
|
|
/* already closed */
|
|
}
|
|
resolve(val);
|
|
};
|
|
|
|
const timer = setTimeout(
|
|
() => finish({ ok: false, reason: 'timeout', detail: `no reply within ${config.rcon.timeoutMs}ms` }),
|
|
config.rcon.timeoutMs
|
|
);
|
|
|
|
sock.on('message', (msg) => {
|
|
clearTimeout(timer);
|
|
finish({ ok: true, buf: msg });
|
|
});
|
|
sock.on('error', (err) => {
|
|
clearTimeout(timer);
|
|
finish({ ok: false, reason: 'send-error', detail: err.message });
|
|
});
|
|
|
|
sock.send(payload, 0, payload.length, port, host, (err) => {
|
|
if (err) {
|
|
clearTimeout(timer);
|
|
finish({ ok: false, reason: 'send-error', detail: err.message });
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
interface EndpointResolution {
|
|
endpoint: RconEndpoint | null;
|
|
// Present when endpoint is null — tells the caller why we couldn't dial.
|
|
reason?: 'unknown-server' | 'no-port-no-komodo' | 'no-port-komodo-empty';
|
|
}
|
|
|
|
/**
|
|
* Resolve the RCON endpoint (host + port) for a server, filling
|
|
* `servers.game_port` from Komodo on first miss.
|
|
*/
|
|
async function endpointFor(server: string): Promise<EndpointResolution> {
|
|
const row = getServer(server);
|
|
if (!row) return { endpoint: null, reason: 'unknown-server' };
|
|
let port = row.game_port;
|
|
if (!port) {
|
|
if (!komodoConfigured()) {
|
|
return { endpoint: null, reason: 'no-port-no-komodo' };
|
|
}
|
|
port = await komodoResolveGamePort(server);
|
|
if (port) setServerField(server, 'game_port', port);
|
|
}
|
|
if (!port) return { endpoint: null, reason: 'no-port-komodo-empty' };
|
|
return { endpoint: { host: row.rcon_host ?? config.rcon.host, port } };
|
|
}
|
|
|
|
function stripPrefix(buf: Buffer): string | null {
|
|
if (buf.length < PREFIX.length) return null;
|
|
for (let i = 0; i < PREFIX.length; i++) if (buf[i] !== PREFIX[i]) return null;
|
|
return buf.subarray(PREFIX.length).toString('utf8');
|
|
}
|
|
|
|
/**
|
|
* Count the number of player rows in a `status` response. The `print`
|
|
* response body looks like:
|
|
*
|
|
* print
|
|
* map: mp_kino
|
|
* num score ping guid name
|
|
* --- ----- ---- -------- ----
|
|
* 0 12 45 abc123...def joe
|
|
* 1 0 72 xyz789...abc jill
|
|
*
|
|
* We look for the `--- ---` separator and count non-blank lines after it.
|
|
* Falls through to a statusResponse parse if the payload uses that dialect.
|
|
*/
|
|
export function parsePlayerCount(text: string): number | null {
|
|
const lines = text.split(/\r?\n/);
|
|
const sep = lines.findIndex((l) => /^-+\s+-+/.test(l.trim()));
|
|
if (sep !== -1) {
|
|
let n = 0;
|
|
for (let i = sep + 1; i < lines.length; i++) {
|
|
const line = lines[i].trim();
|
|
if (!line) continue;
|
|
// Player rows begin with the client num (digits) then whitespace.
|
|
if (/^\d+\s+/.test(line)) n++;
|
|
}
|
|
return n;
|
|
}
|
|
// statusResponse dialect: info line starts with '\', players start on
|
|
// the line after. Same idea — count non-blank lines past the info.
|
|
const info = lines.findIndex((l) => l.startsWith('\\'));
|
|
if (info === -1) return null;
|
|
let n = 0;
|
|
for (let i = info + 1; i < lines.length; i++) {
|
|
if (lines[i].trim()) n++;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
/**
|
|
* Parse `\key\value\key\value…` from a statusResponse info line.
|
|
*/
|
|
export function parseInfoLine(text: string): Record<string, string> {
|
|
const info = text.split(/\r?\n/).find((l) => l.startsWith('\\'));
|
|
if (!info) return {};
|
|
const out: Record<string, string> = {};
|
|
const parts = info.split('\\').filter(Boolean);
|
|
for (let i = 0; i < parts.length - 1; i += 2) out[parts[i]] = parts[i + 1];
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Parse `key: value` header lines from a `status` (print) response, stopping
|
|
* at the scoreboard `--- ---` separator so player rows (which can contain
|
|
* colons in guids or names) don't leak into the metadata. Plutonium T5's
|
|
* `status` puts things like `map: mp_kino` and (on some builds) gametype
|
|
* in the header.
|
|
*/
|
|
export function parseStatusHeader(text: string): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
for (const rawLine of text.split(/\r?\n/)) {
|
|
const line = rawLine.trim();
|
|
if (/^-+\s+-+/.test(line)) break; // scoreboard row — stop
|
|
if (/^\d+\s+/.test(line)) break; // player row that snuck in
|
|
const m = /^([a-z][a-z0-9_]*)\s*:\s*(.+)$/i.exec(line);
|
|
if (m) out[m[1].toLowerCase()] = m[2].trim();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export interface RconPlayer {
|
|
num: number | null; // client slot — print dialect only
|
|
score: number;
|
|
ping: number;
|
|
guid: string | null; // format varies by engine (hex, decimal, steam ID)
|
|
name: string;
|
|
}
|
|
|
|
/**
|
|
* Strip Plutonium/CoD color codes from a player name (`^0`..`^9`).
|
|
* These are display-only escape sequences and confuse the UI if left in.
|
|
*/
|
|
export function stripColorCodes(s: string): string {
|
|
return s.replace(/\^\d/g, '');
|
|
}
|
|
|
|
interface Column {
|
|
name: string;
|
|
start: number;
|
|
end: number; // exclusive
|
|
}
|
|
|
|
/**
|
|
* Detect column boundaries from a header line by locating each non-whitespace
|
|
* word. Column N spans from word N's start to word N+1's start (last column
|
|
* runs to EOL), which is how CoD status output actually lays out — values
|
|
* can borrow the padding space between adjacent columns.
|
|
*/
|
|
function detectHeaderColumns(headerLine: string): Column[] {
|
|
const words: Array<{ name: string; start: number }> = [];
|
|
const re = /\S+/g;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(headerLine)) !== null) {
|
|
words.push({ name: m[0].toLowerCase(), start: m.index });
|
|
}
|
|
return words.map((w, idx) => ({
|
|
name: w.name,
|
|
start: w.start,
|
|
end: idx + 1 < words.length ? words[idx + 1].start : Number.MAX_SAFE_INTEGER
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Extract every player row from a `status` (print) or `getstatus`
|
|
* (statusResponse) response. Returns [] when the payload has none.
|
|
*
|
|
* `status` (print dialect) — parsed as fixed-width based on the header
|
|
* word positions. Both the old 5-column format
|
|
*
|
|
* num score ping guid name
|
|
* --- ----- ---- -------- ----
|
|
* 0 12 45 abc123...def joe
|
|
*
|
|
* and Plutonium T5's 9-column format
|
|
*
|
|
* num score ping guid name lastmsg address qport rate
|
|
* --- ----- ---- ------- ---------------- ------- --------------------- ----- -----
|
|
* 0 3040 36 2741626 JoshWright^7 0 68.235.46.100:28960 -14733 25000
|
|
*
|
|
* work through the same code — column boundaries come from where the
|
|
* header words start, so trailing columns don't get glued onto the name.
|
|
*
|
|
* `getstatus` (statusResponse dialect) — Quake3-style `score ping "name"`.
|
|
*/
|
|
export function parsePlayerRoster(text: string): RconPlayer[] {
|
|
const lines = text.split(/\r?\n/);
|
|
const sep = lines.findIndex((l) => /^-+\s+-+/.test(l.trim()));
|
|
if (sep > 0) {
|
|
const headerLine = lines[sep - 1];
|
|
const cols = detectHeaderColumns(headerLine);
|
|
if (cols.length < 4) return [];
|
|
|
|
const idxOf = (name: string): number => cols.findIndex((c) => c.name === name);
|
|
const numI = idxOf('num');
|
|
const scoreI = idxOf('score');
|
|
const pingI = idxOf('ping');
|
|
const guidI = idxOf('guid');
|
|
const nameI = idxOf('name');
|
|
if (numI < 0 || scoreI < 0 || pingI < 0 || nameI < 0) return [];
|
|
|
|
const slice = (line: string, colIdx: number): string => {
|
|
const c = cols[colIdx];
|
|
return line.slice(c.start, c.end).trim();
|
|
};
|
|
|
|
const out: RconPlayer[] = [];
|
|
for (let i = sep + 1; i < lines.length; i++) {
|
|
const line = lines[i];
|
|
if (!line.trim()) continue;
|
|
const numStr = slice(line, numI);
|
|
if (!/^\d+$/.test(numStr)) continue; // skip stray non-player lines
|
|
const num = Number(numStr);
|
|
const score = Number(slice(line, scoreI));
|
|
const ping = Number(slice(line, pingI));
|
|
if (!Number.isFinite(num) || !Number.isFinite(score) || !Number.isFinite(ping)) continue;
|
|
|
|
const guid = guidI >= 0 ? slice(line, guidI) || null : null;
|
|
const name = stripColorCodes(slice(line, nameI));
|
|
|
|
out.push({ num, score, ping, guid, name });
|
|
}
|
|
return out;
|
|
}
|
|
// statusResponse dialect: rows after the info line, format: score ping "name"
|
|
const info = lines.findIndex((l) => l.startsWith('\\'));
|
|
if (info === -1) return [];
|
|
const out: RconPlayer[] = [];
|
|
const re = /^\s*(-?\d+)\s+(\d+)\s+"(.*)"\s*$/;
|
|
for (let i = info + 1; i < lines.length; i++) {
|
|
const m = re.exec(lines[i]);
|
|
if (!m) continue;
|
|
out.push({
|
|
num: null,
|
|
score: Number(m[1]),
|
|
ping: Number(m[2]),
|
|
guid: null,
|
|
name: stripColorCodes(m[3])
|
|
});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export async function rconGetPlayerCount(server: string): Promise<number | null> {
|
|
const result = await rconStatusResult(server);
|
|
return result.ok ? result.status.players : null;
|
|
}
|
|
|
|
export interface RconStatus {
|
|
players: number;
|
|
capacity?: number | null;
|
|
map: string | null;
|
|
gametype: string | null;
|
|
roster: RconPlayer[];
|
|
}
|
|
|
|
export type RconStatusResult =
|
|
| {
|
|
ok: true;
|
|
endpoint: RconEndpoint;
|
|
status: RconStatus;
|
|
// Kept on success too, so operators can compare "what we parsed"
|
|
// vs "what the server actually sent" from the detail page.
|
|
rawReply: string;
|
|
}
|
|
| {
|
|
ok: false;
|
|
endpoint: RconEndpoint | null;
|
|
reason: RconFailure;
|
|
detail: string;
|
|
// Present on bad-reply: printable hex + ASCII dump of the raw UDP
|
|
// payload so the operator can see what the server actually said
|
|
// (wrong password body, unexpected dialect, etc).
|
|
rawReply?: string;
|
|
};
|
|
|
|
const RAW_REPLY_LIMIT = 512;
|
|
const statusInFlight = new Map<string, Promise<RconStatusResult>>();
|
|
|
|
/** Retry only this read-only query: commands with effects must never be replayed. */
|
|
async function sendStatus(endpoint: RconEndpoint): Promise<SendResult> {
|
|
for (let attempt = 0; ; attempt++) {
|
|
const result = await sendRcon(endpoint.host, endpoint.port, 'status');
|
|
if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) {
|
|
return result;
|
|
}
|
|
await new Promise<void>(resolve => setTimeout(resolve, 250 * 2 ** attempt));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Render a Buffer as a short hex + printable-ASCII snippet — same layout as
|
|
* `hexdump -C`, capped at RAW_REPLY_LIMIT bytes so it fits in the UI panel.
|
|
*/
|
|
export function formatRawReply(buf: Buffer): string {
|
|
const bytes = buf.subarray(0, RAW_REPLY_LIMIT);
|
|
const lines: string[] = [];
|
|
for (let i = 0; i < bytes.length; i += 16) {
|
|
const chunk = bytes.subarray(i, i + 16);
|
|
const hex = Array.from(chunk).map((b) => b.toString(16).padStart(2, '0')).join(' ');
|
|
const ascii = Array.from(chunk).map((b) => (b >= 0x20 && b < 0x7f ? String.fromCharCode(b) : '.')).join('');
|
|
lines.push(`${i.toString(16).padStart(4, '0')} ${hex.padEnd(47, ' ')} ${ascii}`);
|
|
}
|
|
if (buf.length > RAW_REPLY_LIMIT) {
|
|
lines.push(`… (${buf.length - RAW_REPLY_LIMIT} more bytes)`);
|
|
}
|
|
return lines.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Full RCON status probe with structured diagnostics. Callers get either
|
|
* (ok, endpoint, status) or (ok:false, endpoint?, reason, detail). This is
|
|
* the shape the fleet UI consumes to explain *why* a server is unreachable
|
|
* rather than just "unreachable".
|
|
*/
|
|
export async function rconStatusResult(server: string): Promise<RconStatusResult> {
|
|
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
|
|
if (native) {
|
|
const key = JSON.stringify(['agent', native.id, native.generation, latestGameSubmission(server)]);
|
|
const existing = statusInFlight.get(key);
|
|
if (existing) return existing;
|
|
const endpoint = { host: `agent:${native.host_id}`, port: native.port };
|
|
const request = (async (): Promise<RconStatusResult> => {
|
|
try {
|
|
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' });
|
|
const players = parsePlayerCount(reply.text);
|
|
if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
|
|
const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
|
|
return { ok: true, endpoint, status: { players,
|
|
capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
|
|
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
|
|
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) }, rawReply: reply.text.slice(0, 4000) };
|
|
} catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; }
|
|
})().finally(() => statusInFlight.delete(key));
|
|
statusInFlight.set(key, request); return request;
|
|
}
|
|
// Snapshot before any network work. A probe begun after a restart or a
|
|
// replacement registration must not join one observing the old process.
|
|
const createdAt = getServer(server)?.created_at;
|
|
const epoch = createdAt ? latestGameSubmission(server) : null;
|
|
const res = await endpointFor(server);
|
|
if (!res.endpoint) {
|
|
if (res.reason === 'unknown-server') {
|
|
return { ok: false, endpoint: null, reason: 'no-port', detail: `unknown server: ${server}` };
|
|
}
|
|
if (res.reason === 'no-port-no-komodo') {
|
|
return {
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'no-port',
|
|
detail: 'no game_port set and Komodo is not configured'
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'no-port',
|
|
detail: 'no game_port — Komodo did not return one for this server'
|
|
};
|
|
}
|
|
|
|
const ep = res.endpoint;
|
|
const key = JSON.stringify([server, createdAt, epoch, ep.host, ep.port]);
|
|
const pending = statusInFlight.get(key);
|
|
if (pending) return pending;
|
|
const request = probeStatus(ep).finally(() => { statusInFlight.delete(key); });
|
|
statusInFlight.set(key, request);
|
|
return request;
|
|
}
|
|
|
|
async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
|
|
// Plutonium T5 doesn't recognize `getstatus` as an rcon subcommand
|
|
// ("Unknown command"), so we use `status` — a print-body response with
|
|
// a header table plus one row per player. Q3-derived engines that
|
|
// *do* answer statusResponse (via anonymous `getstatus` query) would
|
|
// give richer info-line dvars, but every engine we care about supports
|
|
// `status` under the rcon wrapper.
|
|
const send = await sendStatus(ep);
|
|
if (!send.ok) {
|
|
return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail };
|
|
}
|
|
|
|
const text = stripPrefix(send.buf);
|
|
if (!text) {
|
|
return {
|
|
ok: false,
|
|
endpoint: ep,
|
|
reason: 'bad-reply',
|
|
detail: `reply missing 0xff prefix (${send.buf.length} bytes) — wrong port or non-Plutonium listener?`,
|
|
rawReply: formatRawReply(send.buf)
|
|
};
|
|
}
|
|
const info = parseInfoLine(text);
|
|
const players = parsePlayerCount(text);
|
|
if (players === null) {
|
|
// Reply arrived but neither the print separator nor an info line
|
|
// showed up. Common wrong-password wordings vary across Plutonium
|
|
// builds ("Bad rcon_password", "Invalid password", "Bad password").
|
|
const trimmed = text.trim();
|
|
const looksLikePrint = trimmed.startsWith('print');
|
|
const suggestsPassword = /(bad|invalid|wrong).{0,20}(rcon|password|pass)/i.test(text);
|
|
let detail: string;
|
|
if (looksLikePrint && suggestsPassword) {
|
|
detail = 'wrong RCON password';
|
|
} else if (looksLikePrint) {
|
|
// print body but we couldn't classify — show the first line
|
|
const firstLine = trimmed.split(/\r?\n/)[1]?.trim() ?? '';
|
|
detail = firstLine
|
|
? `print reply not recognized: "${firstLine.slice(0, 100)}"`
|
|
: `unparseable print reply (${text.length} bytes)`;
|
|
} else {
|
|
detail = `unparseable reply (${send.buf.length} bytes) — see snippet below`;
|
|
}
|
|
return { ok: false, endpoint: ep, reason: 'bad-reply', detail, rawReply: formatRawReply(send.buf) };
|
|
}
|
|
const header = parseStatusHeader(text);
|
|
return {
|
|
ok: true,
|
|
endpoint: ep,
|
|
status: {
|
|
// Prefer statusResponse `\info\` line values when present, fall
|
|
// back to print-body `key: value` header lines otherwise.
|
|
players,
|
|
capacity: positiveCapacity(info.sv_maxclients ?? info.sv_maxClients ?? header.sv_maxclients ?? header.sv_maxClients),
|
|
map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
|
|
gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null,
|
|
roster: parsePlayerRoster(text)
|
|
},
|
|
rawReply: formatRawReply(send.buf)
|
|
};
|
|
}
|
|
|
|
function positiveCapacity(value: string | undefined): number | null {
|
|
const count = Number(value);
|
|
return Number.isInteger(count) && count > 0 ? count : null;
|
|
}
|
|
|
|
/** Query the running process, not the updater's on-disk files. */
|
|
export async function rconPlutoniumVersion(server: string): Promise<{ revision: number | null; error: string | null }> {
|
|
const native = getServer(server)?.native_instance_id ? nativeInstance(server) : null;
|
|
if (native) {
|
|
try {
|
|
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'version' });
|
|
const revision = parsePlutoniumRevision(reply.text);
|
|
return { revision, error: revision === null ? 'Agent RCON reply did not identify a runtime revision.' : null };
|
|
} catch (error) { return { revision: null, error: (error as Error).message }; }
|
|
}
|
|
const { endpoint } = await endpointFor(server);
|
|
if (!endpoint) return { revision: null, error: 'No RCON endpoint available.' };
|
|
const response = await sendRcon(endpoint.host, endpoint.port, 'version');
|
|
if (!response.ok) return { revision: null, error: response.detail };
|
|
const body = stripPrefix(response.buf);
|
|
const revision = body?.trimStart().startsWith('print') ? parsePlutoniumRevision(body) : null;
|
|
return { revision, error: revision === null ? 'RCON version reply did not identify a Plutonium revision.' : null };
|
|
}
|