josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Read chat logs incrementally and drain retention (Refs #62)

Browse Source
This commit is contained in:
josh
2026-10-08 17:28:49 -04:00
parent 0cc4498b53
commit c10f602e8e
18 changed files with 646 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/chat.ts
+98 -8
View File
@@ -4,26 +4,38 @@ import { enrichGameChatTimeline, type ChatMatch, type ChatRoundReport, type Cont
import { parseGameEvent } from '../../telemetry.js';
import { telemetryRetention, telemetryCutoff } from './telemetry-requests.js';
import { getServer, listServers, type ServerRow } from './servers.js';
import { gameLogKey, resolveLogPath, tailLog, type TailResult } from './logs.js';
import { gameLogKey, resolveLogPath, readLogChunk, type LogCursor, type TailResult } from './logs.js';
import { listHosts } from '../hosts/index.js';
import { observationReader } from './status.js';
/** Historical context is scoped to this registration, including round-one matches. */
export function serverChatTimeline(server: ServerRow, tail: TailResult): ContextualChatEntry[] {
const unscoped = enrichGameChatTimeline(tail.lines, { mtime: tail.mtime, matches: [], reports: [] });
const times = unscoped.flatMap(entry => entry.context.realTime ? [entry.context.realTime] : []);
if (!times.length) return unscoped;
const since = times.reduce((a, b) => a < b ? a : b), until = times.reduce((a, b) => a > b ? a : b);
const handle = db();
const matches = handle.prepare(`SELECT id, started_at AS startedAt, ended_at AS endedAt,
last_received_at AS lastReceivedAt, status, start_observed AS startObserved
FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL
ORDER BY id DESC LIMIT 2000`).all(server.name) as Array<Omit<ChatMatch, 'startObserved'> & { startObserved: number }>;
AND started_at <= ? AND (COALESCE(ended_at,last_received_at) >= ? OR status='active')
ORDER BY id DESC LIMIT 2000`).all(server.name, until, since) as Array<Omit<ChatMatch, 'startObserved'> & { startObserved: number }>;
// Diagnostic history survives deletion. Its receipt time must belong to the
// current registration so a reused server name cannot inherit old rounds.
const cutoff = telemetryCutoff();
const roundSince = new Date(Date.parse(since) - 120_000).toISOString();
const matchSince = matches.reduce((earliest, match) => match.startedAt < earliest ? match.startedAt : earliest, since);
const reports = handle.prepare(`SELECT received_at AS receivedAt, event, round,
player_count AS playerCount FROM telemetry_requests
WHERE server = ? AND status = 200 AND received_at >= ?
AND julianday(received_at) >= julianday(?)
AND (received_at >= ? OR (event='game_start' AND received_at >= ?))
AND received_at <= ?
AND event IN ('game_start', 'round_change', 'heartbeat', 'game_end')
AND player_count >= 0 AND (event = 'game_start' OR round >= 0)
ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at,
ORDER BY id DESC LIMIT ?`).all(server.name, cutoff, server.created_at, roundSince, matchSince,
new Date(Date.parse(until) + 120_000).toISOString(),
(telemetryRetention().maxPerServer || 10000)) as ChatRoundReport[];
// Saving gameplay does not depend on diagnostic logging succeeding. Keep the
@@ -77,6 +89,10 @@ function archiveChat(entries: ContextualChatEntry[]): void {
export function currentMatchChat(server: ServerRow, tail: TailResult | null): ContextualChatEntry[] {
const entries = tail ? serverChatTimeline(server, tail) : [];
archiveChat(entries);
return currentChatEntries(server, entries);
}
function currentChatEntries(server: ServerRow, entries: ContextualChatEntry[]): ContextualChatEntry[] {
const match = db().prepare(`SELECT id, started_at AS startedAt FROM leaderboard_matches
WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1`)
.get(server.name) as { id: number; startedAt: string } | undefined;
@@ -101,6 +117,12 @@ export function currentMatchChat(server: ServerRow, tail: TailResult | null): Co
}
const pending = new Map<string, Promise<{ entries: ContextualChatEntry[]; missing: boolean }>>();
interface ChatReadState {
key: string; cursor?: LogCursor; lines: string[]; partial: Buffer; skipLine: boolean;
signature?: string; entries: ContextualChatEntry[]; contextVersion?: string; wasOccupied: boolean;
}
const readers = new Map<string, ChatReadState>();
const failures = new Map<string, number>();
/** Shared by the live API and the worker; collection does not require a viewer. */
export function collectServerChat(server: ServerRow): Promise<{ entries: ContextualChatEntry[]; missing: boolean }> {
@@ -110,26 +132,94 @@ export function collectServerChat(server: ServerRow): Promise<{ entries: Context
const work = (async () => {
const path = resolveLogPath(server.name, gameLogKey(server.name));
if (!path) throw new Error('Invalid server log path');
const tail = await tailLog(path, 2000);
let saved = readers.get(server.name);
if (saved?.key !== key) saved = undefined;
// Publish the new cursor only after its messages have been processed.
const state: ChatReadState = saved ? { ...saved } : { key, lines: [], partial: Buffer.alloc(0), skipLine: false, entries: [], wasOccupied: false };
const tail = await readLogChunk(path, state.cursor);
const current = getServer(server.name);
// A remote read may finish after destruction or replacement.
if (!current || current.destroyed_at || current.created_at !== server.created_at
|| current.native_instance_id !== server.native_instance_id || current.instance_generation !== server.instance_generation) {
return { entries: [], missing: true };
}
return { entries: currentMatchChat(current, tail), missing: tail === null };
if (!tail) {
readers.delete(server.name);
return { entries: currentChatEntries(current, []), missing: true };
}
let changed = false;
if ('cursor' in tail) {
if (tail.reset) { state.lines = []; state.partial = Buffer.alloc(0); state.skipLine = tail.truncated; state.entries = []; }
state.cursor = tail.cursor;
const bytes = Buffer.concat([state.partial, Buffer.from(tail.contentBase64, 'base64')]);
const end = bytes.lastIndexOf(10) + 1;
if (end) {
const lines = bytes.subarray(0, end).toString('utf8').split('\n').slice(0, -1).map(line => line.replace(/\r$/, ''));
if (state.skipLine) lines.shift();
state.skipLine = false;
state.lines = [...state.lines, ...lines].slice(-2000);
let length = 0, start = state.lines.length;
while (start > 0 && (length += Buffer.byteLength(state.lines[start - 1]) + 1) <= 256 * 1024) start--;
state.lines = state.lines.slice(start);
changed = true;
}
state.partial = Buffer.from(bytes.subarray(end));
if (state.partial.length > 256 * 1024) { state.partial = Buffer.alloc(0); state.skipLine = true; }
// File mtime anchors the final line only after the backlog has been read.
if (tail.cursor.offset < tail.size || state.partial.length) {
state.signature = undefined;
readers.set(server.name, state);
return { entries: currentChatEntries(current, state.entries), missing: false };
}
} else {
// Older agents remain readable during a rolling update.
changed = state.signature !== `${tail.size}:${tail.mtime}`;
if (changed) state.lines = tail.lines;
}
const signature = `${tail.size}:${tail.mtime}`;
const needsContext = state.entries.some(entry => entry.type === 'message' && entry.matchId === undefined);
const contextVersion = needsContext ? (db().prepare('SELECT received_at FROM server_telemetry WHERE server=?').get(server.name) as { received_at: string } | undefined)?.received_at : undefined;
if (changed || state.signature === undefined || (needsContext && contextVersion !== state.contextVersion)) {
state.entries = serverChatTimeline(current, { ...tail, lines: state.lines });
archiveChat(state.entries);
state.contextVersion = contextVersion;
}
state.signature = signature;
readers.set(server.name, state);
return { entries: currentChatEntries(current, state.entries), missing: false };
})().finally(() => pending.delete(key));
pending.set(key, work);
return work;
}
export async function collectFleetChat(): Promise<void> {
const servers = listServers().filter(server => !server.destroyed_at);
const hosts = new Map(listHosts().map(host => [host.id, host]));
const observe = observationReader();
const live = listServers().filter(server => !server.destroyed_at);
const names = new Set(live.map(server => server.name));
for (const name of readers.keys()) if (!names.has(name)) readers.delete(name);
for (const name of failures.keys()) if (!names.has(name)) failures.delete(name);
const servers = live.filter(server => {
if (server.host_id && !hosts.get(server.host_id)?.connected) return false;
if (server.container_state && !['running', 'unknown'].includes(server.container_state)) return false;
const players = observe(server).last_players ?? server.last_players;
// One final read after players leave retains messages written at match end.
return players !== 0 || readers.get(server.name)?.wasOccupied;
});
// Bound agent traffic and isolate failures to their own server.
async function collect() {
for (let server = servers.shift(); server; server = servers.shift()) {
try { await collectServerChat(server); }
catch (error) { console.error(`[chat] Could not collect chat for ${server.name}`, error); }
try {
await collectServerChat(server);
const state = readers.get(server.name);
if (state) state.wasOccupied = (observe(server).last_players ?? server.last_players ?? 0) > 0;
failures.delete(server.name);
} catch (error) {
if (Date.now() - (failures.get(server.name) ?? -Infinity) >= 5 * 60_000) {
console.error(`[chat] Could not collect chat for ${server.name}`, error);
failures.set(server.name, Date.now());
}
}
}
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, collect));
src/lib/server/fleet/logs.ts
+44 -1
View File
@@ -1,7 +1,8 @@
import { open, stat } from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { resolve, sep } from 'node:path';
import { rootOfServer, gameOfServer } from '../config.js';
import { nativeTail } from '../backends/files.js';
import { nativeTail, nativeLogChunk } from '../backends/files.js';
// Whitelisted log files exposed to the UI, mapped to their path relative
// to each server's root directory. Adding a new supported log = one entry
@@ -48,6 +49,48 @@ export interface TailResult {
truncated: boolean; // true when we didn't read from the start of the file
}
export interface LogCursor {
fileId: string; offset: number; size: number; modified: string;
anchorLength: number; anchorHash: string;
}
export interface LogChunk {
contentBase64: string; size: number; mtime: string; truncated: boolean;
cursor: LogCursor; reset: boolean;
}
/** Bounded byte reads; an anchor before the offset detects copy-truncate and regrowth. */
export async function readLogChunk(path: string, cursor?: LogCursor): Promise<LogChunk | TailResult | null> {
const remote = await nativeLogChunk(path, cursor);
if (remote !== undefined) return remote;
let file;
try { file = await open(path, 'r'); }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; }
try {
const st = await file.stat();
if (!st.isFile()) return null;
const fileId = `${st.dev}:${st.ino}`, modified = `${st.mtimeMs}:${st.ctimeMs}`, size = st.size;
const mtime = st.mtime.toISOString();
if (cursor?.fileId === fileId && cursor.modified === modified && cursor.size === size && cursor.offset === size) {
return { contentBase64: '', size, mtime, truncated: cursor.offset > 0, cursor, reset: false };
}
const hash = (value: Buffer) => createHash('sha256').update(value).digest('hex');
let reset = !cursor || cursor.fileId !== fileId || cursor.offset > size;
if (cursor && !reset) {
const anchor = Buffer.alloc(cursor.anchorLength);
const read = await file.read(anchor, 0, anchor.length, cursor.offset - anchor.length);
reset = read.bytesRead !== anchor.length || hash(anchor) !== cursor.anchorHash;
}
const start = reset ? Math.max(0, size - READ_WINDOW_BYTES) : cursor!.offset;
const content = Buffer.alloc(Math.min(READ_WINDOW_BYTES, size - start));
const read = await file.read(content, 0, content.length, start);
const offset = start + read.bytesRead;
const anchor = Buffer.alloc(Math.min(256, offset));
await file.read(anchor, 0, anchor.length, offset - anchor.length);
return { contentBase64: content.subarray(0, read.bytesRead).toString('base64'), size, mtime, truncated: start > 0, reset,
cursor: { fileId, modified, size, offset, anchorLength: anchor.length, anchorHash: hash(anchor) } };
} finally { await file.close(); }
}
const DEFAULT_TAIL_LINES = 200;
const MAX_TAIL_LINES = 2000;
const READ_WINDOW_BYTES = 256 * 1024; // 256 KB — plenty for a few hundred lines