josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Improve application responsiveness and background processing

Browse Source
This commit is contained in:
josh
2026-10-04 19:56:06 -04:00
parent 1392a082da
commit ed004ddbcf
70 changed files with 1324 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/index.ts
+2
View File
@@ -1,4 +1,5 @@
import Database from 'better-sqlite3';
import { installReportRevisions } from '../reports/revisions.js';
import { mkdirSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { config } from '../config.js';
@@ -368,6 +369,7 @@ export function db(): Database.Database {
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrateNativeInstances(handle);
installReportRevisions(handle);
} catch (err) {
handle.close();
throw err;
src/lib/server/fleet/leaderboard.ts
+33 -24
View File
@@ -10,6 +10,11 @@ const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
export const MAX_LEADERBOARD_OFFSET = 1_000_000;
type RankedEntry = Omit<LeaderboardEntry, 'records'>;
interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
// A left join keeps metadata available when the requested page has no players.
guid: string | null;
@@ -91,21 +96,15 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
SELECT measures.*, p.name, RANK() OVER (ORDER BY measures.${metric} DESC) AS rank
FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
)`;
if (exactGuid !== undefined) {
const entries = handle.prepare(`${aggregated} SELECT * FROM ranked WHERE guid = ?`)
.all(...parameters, exactGuid) as RankedEntry[];
return addRecordSources(handle, entries, options)[0] ?? null;
}
const query = options.q?.trim() ?? '';
const search = query ? 'WHERE instr(lower(name), lower(?)) > 0 OR instr(lower(guid), lower(?)) > 0' : '';
// Reuse one ranked set for the page and matching count. Summaries deliberately
// cover the full game/map scope, including players below an efficiency threshold.
const rows = handle.prepare(`${aggregated}, searched AS MATERIALIZED (
SELECT * FROM ranked ${search}
), page AS (
SELECT * FROM searched ORDER BY ${metric} DESC, name COLLATE NOCASE, name, guid LIMIT ? OFFSET ?
), playerSummary AS (
let revision = -1;
try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
catch { /* Standalone history tools can operate without cache metadata. */ }
let cache = scopeCaches.get(handle);
if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', metric]);
let saved = cache.get(cacheKey);
if (!saved || revision < 0 || saved.revision !== revision) {
const rows = handle.prepare(`${aggregated}, playerSummary AS (
SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -113,21 +112,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
SELECT COUNT(DISTINCT match_id) AS matches, MIN(matchStartedAt) AS firstTrackedAt,
MAX(matchUpdatedAt) AS updatedAt FROM observations
), metadata AS (
SELECT (SELECT COUNT(*) FROM searched) AS total, playerSummary.*, matchSummary.*
FROM playerSummary CROSS JOIN matchSummary
) SELECT page.*, metadata.total AS matchingTotal, metadata.players AS summaryPlayers,
SELECT playerSummary.*, matchSummary.* FROM playerSummary CROSS JOIN matchSummary
) SELECT ranked.*, metadata.players AS summaryPlayers,
metadata.matches AS summaryMatches, metadata.kills AS summaryKills,
metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
FROM metadata LEFT JOIN page ON 1
ORDER BY page.${metric} DESC, page.name COLLATE NOCASE, page.name, page.guid`)
.all(...parameters, ...(query ? [query, query] : []), limit, offset) as LeaderboardRow[];
FROM metadata LEFT JOIN ranked ON 1
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
const entries = addRecordSources(handle, rankedEntries, options);
const metadata = rows[0];
const total = metadata.matchingTotal;
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
@@ -140,5 +136,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
GROUP BY game, map_key HAVING id = MAX(id)
ORDER BY game, name COLLATE NOCASE, map_key`)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
return { entries, maps, total, limit, offset, metric, summary };
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
}
if (exactGuid !== undefined) {
const entry = saved.entries.find(entry => entry.guid === exactGuid);
return entry ? addRecordSources(handle, [entry], options)[0] : null;
}
const query = searchText(options.q?.trim() ?? '');
const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
}
src/lib/server/fleet/servers.ts
+10 -2
View File
@@ -179,6 +179,10 @@ export function setServerField(
field: keyof ServerRow,
value: string | number | null
): void {
setServerFields(name, { [field]: value });
}
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
const allowed: Array<keyof ServerRow> = [
'komodo_type',
'komodo_name',
@@ -211,8 +215,12 @@ export function setServerField(
'game_ended_at',
'game_cycle_key'
];
if (!allowed.includes(field)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${field} = ? WHERE name = ?`).run(value, name);
const entries = Object.entries(fields);
if (!entries.length) return;
for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
.run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
}
/**
src/lib/server/fleet/snapshot.ts
+51
View File
@@ -0,0 +1,51 @@
import { db } from '../db/index.js';
import { listServers } from './servers.js';
import { observationReader } from './status.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { runningVersions } from '../integrations/plutonium.js';
import { listHosts } from '../hosts/index.js';
import type { NativeInstance } from '../backends/native.js';
export function fleetSnapshot() {
const rconHost = config.rcon.host;
const hosts = listHosts();
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
const versions = new Map(runningVersions().map(version => [version.server, version]));
const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
const holds = new Map<string, { target: string; betaId: number }[]>();
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
{ server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
}
const observe = observationReader();
const servers = listServers().map((s) => {
const instance = s.native_instance_id ? instances.get(s.name) : null;
return {
...observe(s),
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: holds.get(s.name) ?? []
};
});
return {
servers,
serverNow: Date.now(),
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
komodo: komodoConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
defaultRconHost: rconHost
}
};
}
src/lib/server/fleet/status.ts
+41 -5
View File
@@ -4,21 +4,57 @@ import { isProvisioning } from './provisioning.js';
import { restartSettling } from '../provisioning/restarts.js';
import { type ServerRow } from './servers.js';
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { restartRequestInFlight } from '../provisioning/restarts.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
/** One set of evidence reads per snapshot, rather than four queries per server. */
export function observationReader() {
const handle = db();
const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
{ server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM (
SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
{ server: string; at: string | null }[]).map(row => [row.server, row.at]));
const reports = new Map<string, GameTelemetry>();
for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
{ server: string; received_at: string; payload: string }[]) {
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
} catch { /* Invalid retained telemetry is unavailable. */ }
}
return (server: ServerRow) => {
const operation = operations.get(server.name), restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
});
};
}
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow) {
export function serverObservation(server: ServerRow, evidence?: Evidence) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
const provisioning = isProvisioning(server.name);
const state = restartSettling(server.name) ? 'restarting' : server.container_state;
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
const ready = !provisioning && (!state || state === 'running');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: telemetryReport(server.name) ?? undefined,
telemetry_after: latestGameSubmission(server.name),
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
replySnippet: ready ? server.last_reply_snippet : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
src/lib/server/game-files/index.ts
+16 -9
View File
@@ -18,14 +18,14 @@ type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id,
...(row.bytes_verified == null ? {} : { bytesVerified: row.bytes_verified }),
...(row.transfer_progress ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } : {}) });
function versionFromRow(row: VersionRow): GameLibraryVersion {
const files = JSON.parse(row.manifest) as GameFileEntry[];
function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches?: Map<string, HostLibraryCache[]>): GameLibraryVersion {
const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
description: row.description, state: row.state, sizeBytes: row.size_bytes,
files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
caches: summaryCaches ? summaryCaches.get(row.id) ?? [] : (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
}
export function getLibraryVersion(id: string): GameLibraryVersion | null {
const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
@@ -52,12 +52,19 @@ export function latestPublishedLibraries() {
return version ? [{ ...version, name: game.name }] : [];
});
}
export function listLibraries(): GameLibrary[] {
export function listLibraries(summary = false): GameLibrary[] {
reconcileLibraryOperations();
let summaryCaches: Map<string, HostLibraryCache[]> | undefined;
if (summary) {
summaryCaches = new Map();
for (const row of database().prepare('SELECT * FROM host_library_caches').all() as CacheRow[]) {
const items = summaryCaches.get(row.version_id) ?? []; items.push(cacheFromRow(row)); summaryCaches.set(row.version_id, items);
}
}
const latest = new Map(latestPublishedLibraries().map(version => [version.game, version.id]));
return GAME_FILE_GAMES.map(game => {
const versions = (database().prepare(`SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(versionFromRow);
const versions = (database().prepare(`SELECT ${summary ? `v.id, v.library_id, v.version, v.description, v.dlc, v.state, v.size_bytes, v.manifest_sha256, v.created_at, v.published_at, v.storage_location, v.source_host_id, v.source_volume, v.error, v.operation_id, CASE WHEN json_array_length(v.manifest)>0 THEN json_array_length(v.manifest) ELSE (SELECT COUNT(*) FROM game_file_upload_entries e WHERE e.version_id=v.id) END AS file_count` : 'v.*'}, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(row => versionFromRow(row, summaryCaches));
const firstLibrary = database().prepare('SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game.id) as { created_at: string } | undefined;
return { id: game.id, name: game.name, game: game.id, description: '', createdAt: firstLibrary?.created_at ?? '', latestVersionId: latest.get(game.id) ?? null, versions };
});
@@ -392,9 +399,9 @@ export function reconcileLibraryOperations(): void {
/** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */
export function libraryOverview() {
const hosts = listHosts();
return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
return { libraries: listLibraries(true).map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
...version, caches: hosts.map(host => {
const cached = getHostLibraryCache(host.id, version.id);
const cached = version.caches.find(cache => cache.hostId === host.id);
if (cached) return { ...cached, status: cached.status === 'ready' && library.latestVersionId !== version.id ? 'outdated' as const : cached.status };
return {
hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,
src/lib/server/game-files/uploads.ts
+35 -5
View File
@@ -44,20 +44,50 @@ export function validateUploadPlan(input: unknown): UploadFilePlan[] {
return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
}
const uploadLocks = new Map<string, Promise<void>>();
interface UploadLocks { barrier: Promise<void>; writers: number; files: Map<string, Promise<void>>; active: Set<Promise<void>> }
const uploadLocks = new Map<string, UploadLocks>();
function locksFor(key: string): UploadLocks {
let locks = uploadLocks.get(key);
if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); }
return locks;
}
function releaseLocks(key: string, locks: UploadLocks) {
if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) uploadLocks.delete(key);
}
/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
const key = resolve(root);
const previous = uploadLocks.get(key) ?? Promise.resolve();
const locks = locksFor(key);
// Capture readers admitted before publication, then prevent later readers
// from starting until publication finishes (including failed publication).
const previous = Promise.all([locks.barrier, ...locks.active]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
uploadLocks.set(key, current);
locks.barrier = current;
locks.writers++;
await previous;
try { return await callback(); }
finally {
release();
if (uploadLocks.get(key) === current) uploadLocks.delete(key);
locks.writers--;
releaseLocks(key, locks);
}
}
/** Different files may transfer together; offsets for the same file remain serial. */
export async function withUploadFileLock<T>(root: string, file: string, callback: () => Promise<T>): Promise<T> {
const key = resolve(root), locks = locksFor(key);
const previous = Promise.all([locks.barrier, locks.files.get(file)]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
locks.files.set(file, current); locks.active.add(current);
await previous;
try { return await callback(); }
finally {
release(); locks.active.delete(current);
if (locks.files.get(file) === current) locks.files.delete(file);
releaseLocks(key, locks);
}
}
@@ -95,7 +125,7 @@ export async function receiveUploadChunk(
}
if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
return withUploadLock(root, async () => {
return withUploadFileLock(root, file.path, async () => {
await beforeWrite?.();
const progress = await getUploadFileProgress(root, file);
if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
src/lib/server/hosts/index.ts
+25 -5
View File
@@ -1,4 +1,5 @@
import { recordAgentReceipt } from '../provisioning/agent-events.js';
import { EventEmitter } from 'node:events';
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
import { isIP } from 'node:net';
import { db } from '../db/index.js';
@@ -8,6 +9,16 @@ import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, Hos
export const AGENT_PROTOCOL_VERSION = 1;
export const HEARTBEAT_MAX_AGE_MS = 90_000;
const now = () => new Date().toISOString();
const workEvents = new EventEmitter();
workEvents.setMaxListeners(0);
export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise<void> {
return new Promise(resolve => {
const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); };
const timer = setTimeout(done, timeoutMs);
workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true });
if (signal.aborted) done();
});
}
const digest = (token: string) => createHash('sha256').update(token).digest('hex');
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
@@ -113,6 +124,7 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
.run(id, operationId, digest(token), timestamp, timestamp);
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
queueMicrotask(() => workEvents.emit(id));
return { deleted: false, operationId };
}).immediate();
}
@@ -287,7 +299,7 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
@@ -300,7 +312,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
return database().transaction(() => {
const operation = database().transaction(() => {
if (options.idempotencyKey) {
const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
if (existing) {
@@ -315,8 +327,10 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
return getHostOperation(id)!;
})();
queueMicrotask(() => workEvents.emit(hostId));
return operation;
}
export function pendingHostOperations(hostId: string): HostOperation[] {
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
@@ -328,8 +342,12 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
return database().transaction(() => {
const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
@@ -342,7 +360,7 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
})();
}
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
return database().transaction(() => {
const operation = database().transaction(() => {
requireHost(hostId);
const operation = getHostOperation(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
@@ -367,6 +385,8 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
}
return getHostOperation(id)!;
})();
queueMicrotask(() => workEvents.emit(`operation:${id}`));
return operation;
}
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
const deadline = Date.now() + timeoutMs;
@@ -375,7 +395,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
if (!operation) throw new Error('Host operation was not found.');
if (['completed','failed','unknown'].includes(operation.state)) return operation;
if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
await new Promise(resolve => setTimeout(resolve, 200));
await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
}
}
src/lib/server/integrations/plutonium.ts
+8 -3
View File
@@ -22,13 +22,18 @@ export function freshRelease(): number | null {
export function runningVersions(): PlutoniumVersion[] {
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
const target = freshRelease();
const byServer = new Map(versions.map(version => [version.server, version]));
const epochs = new Map((db().prepare(`SELECT server, MAX(at) AS at FROM (
SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as { server: string; at: string | null }[]).map(row => [row.server, row.at]));
const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
return listServers().map(server => {
const value = versions.find(v => v.server === server.name)
const value = byServer.get(server.name)
?? { server: server.name, revision: null, checked_at: null, error: null };
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
const epoch = latestGameSubmission(server.name);
const epoch = epochs.get(server.name);
const stopped = server.desired_state === 'stopped';
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !restartRecoveryPending(server.name);
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name);
const revision = valid && !stopped ? value.revision : null;
return { ...value, revision, stopped,
update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'
src/lib/server/performance.ts
+13
View File
@@ -0,0 +1,13 @@
import { monitorEventLoopDelay } from 'node:perf_hooks';
let started = false;
export function startPerformanceMonitor() {
if (started || process.env.EC_PERF_LOG !== 'true') return;
started = true;
const delay = monitorEventLoopDelay({ resolution: 20 }); delay.enable();
setInterval(() => {
console.info('[performance]', { eventLoopP95Ms: Math.round(delay.percentile(95) / 1e6),
eventLoopMaxMs: Math.round(delay.max / 1e6), rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
delay.reset();
}, 60_000).unref();
}
src/lib/server/pipeline/bus.ts
+10 -1
View File
@@ -46,6 +46,7 @@ export type BetaStage =
export type BetaState = 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
export type PipelineEvent =
| { type: 'fleet-invalidated' }
| { type: 'server-telemetry'; entries: GameTelemetry[] }
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
| { type: 'server-changed'; deploymentId: string; server: string; files: number }
@@ -58,7 +59,7 @@ export type PipelineEvent =
error: string | null;
}
| { type: 'sweep-tick'; frozen: boolean; reason?: string }
| { type: 'server-status'; entries: ServerStatusEntry[] }
| { type: 'server-status'; entries: ServerStatusEntry[]; delta?: boolean }
| { type: 'beta-opened'; betaId: number; scriptIds: number[]; servers: string[] }
| {
type: 'beta-server-stage';
@@ -78,9 +79,17 @@ class Bus extends EventEmitter {
}
const bus = new Bus();
let invalidationPending = false;
export function invalidateFleet(): void {
if (invalidationPending) return;
invalidationPending = true;
// Flush after nested database transactions have committed.
queueMicrotask(() => { invalidationPending = false; bus.emit('event', { type: 'fleet-invalidated' }); });
}
export function emit(event: PipelineEvent): void {
bus.emit('event', event);
if (event.type === 'beta-opened' || event.type === 'beta-state' || event.type === 'beta-server-stage') invalidateFleet();
}
export function subscribe(listener: (event: PipelineEvent) => void): () => void {
src/lib/server/pipeline/worker.ts
+44 -29
View File
@@ -6,10 +6,10 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation } from '../fleet/status.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
@@ -76,6 +76,15 @@ function encodeError(reason: RconFailure, detail: string): string {
const pendingStatus = new Map<string, Promise<number | null>>();
const statusQueue: Array<() => Promise<void>> = [];
let activeStatus = 0;
const changedStatus = new Map<string, ServerStatusEntry>();
function publishStatus(entry: ServerStatusEntry) {
const scheduled = changedStatus.size > 0;
changedStatus.set(entry.server, entry);
if (!scheduled) queueMicrotask(() => {
const entries = [...changedStatus.values()]; changedStatus.clear();
emit({ type: 'server-status', delta: true, entries });
});
}
function drainStatusQueue(): void {
while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
@@ -95,9 +104,9 @@ function queueStatus(server: string): Promise<number | null> {
let capacity: number | null = null;
try {
capacity = await refreshServerStatus(server);
// Consumers expect a full snapshot. Re-read it now so neither game
// data nor container evidence waits for the slowest fleet member.
emit({ type: 'server-status', entries: listServers().map(statusEntry) });
// Publish this server's latest evidence without waiting for the fleet.
const current = getServer(server);
if (current) publishStatus(statusEntry(current));
} catch (err) { console.error(`[worker] status error for ${server}:`, err); }
finally { pendingStatus.delete(server); finish(capacity); }
});
@@ -113,8 +122,9 @@ export async function fanoutStatus(): Promise<void> {
try {
// Metrics remain complete fleet collections, using freshness at capture
// time rather than the status saved at the beginning of a long scan.
const observe = observationReader();
recordMetrics(listServers().map(srv => {
const current = serverObservation(srv);
const current = observe(srv);
return { server: srv.name, game: srv.game ?? 'unknown',
map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
players: current.health === 'online' ? current.last_players : null,
@@ -127,6 +137,8 @@ async function refreshServerStatus(server: string): Promise<number | null> {
const srv = getServer(server);
if (!srv) return null;
let capacity: number | null = null;
let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const epoch = latestGameSubmission(srv.name);
const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
@@ -136,53 +148,56 @@ async function refreshServerStatus(server: string): Promise<number | null> {
: await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
// A lifecycle request may be submitted while an inspection is in flight.
if (unchanged()) {
setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
setServerField(srv.name, 'container_checked_at', new Date().toISOString());
updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
updates.container_checked_at = new Date().toISOString();
if (!container.ok) {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
updates.last_error = null;
updates.last_reply_snippet = null;
} else if (!telemetryEvidence(srv.name)) {
flush();
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
if (unchanged()) {
const now = new Date().toISOString();
setServerField(srv.name, 'last_checked_at', now);
setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
updates.last_checked_at = now;
updates.last_reply_snippet = result.rawReply ?? null;
if (result.ok) {
capacity = result.status.capacity ?? null;
setServerField(srv.name, 'last_players', result.status.players);
setServerField(srv.name, 'last_map', result.status.map);
setServerField(srv.name, 'last_gametype', result.status.gametype);
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
setServerField(srv.name, 'last_seen_at', now);
setServerField(srv.name, 'last_error', null);
} else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
updates.last_players = result.status.players;
updates.last_map = result.status.map;
updates.last_gametype = result.status.gametype;
updates.last_roster = JSON.stringify(result.status.roster);
updates.last_seen_at = now;
updates.last_error = null;
} else updates.last_error = encodeError(result.reason, result.detail);
}
}
}
} else {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
updates.last_error = null;
updates.last_reply_snippet = null;
}
flush();
const current = getServer(srv.name) ?? srv;
const status = serverObservation(current);
// Occupancy timers and match clearing follow the same source as the UI.
if (status.health === 'online' && status.last_players !== null) {
if (status.last_players > 0) {
setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
setServerField(srv.name, 'inactive_since', null);
updates.active_since = current.active_since ?? status.last_seen_at;
updates.inactive_since = null;
} else {
setServerField(srv.name, 'active_since', null);
setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
setServerField(srv.name, 'game_started_at', null);
setServerField(srv.name, 'game_ended_at', null);
updates.active_since = null;
updates.inactive_since = current.inactive_since ?? status.last_seen_at;
updates.game_started_at = null;
updates.game_ended_at = null;
}
}
flush();
return capacity;
}
function statusEntry(srv: ServerRow): ServerStatusEntry {
const final = serverObservation(srv);
export function statusEntry(srv: ServerRow, observe = serverObservation): ServerStatusEntry {
const final = observe(srv);
return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
lastError: final.last_error, replySnippet: final.replySnippet,
src/lib/server/provisioning/progress.ts
+2
View File
@@ -1,4 +1,5 @@
import { db } from '../db/index.js';
import { invalidateFleet } from '../pipeline/bus.js';
import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck,
type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js';
@@ -116,5 +117,6 @@ export function recordProgress(id: string, update: ProgressUpdate): void {
const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json)
VALUES (?, ?, ?, ?, ?, ?)`);
for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null);
if (events.length) invalidateFleet();
})();
}
src/lib/server/reports/database.ts
+5
View File
@@ -0,0 +1,5 @@
import Database from 'better-sqlite3';
import { workerData } from 'node:worker_threads';
const handle = new Database(workerData.dbPath, { readonly: true, fileMustExist: true });
export function db() { return handle; }
src/lib/server/reports/index.ts
+64
View File
@@ -0,0 +1,64 @@
import { Worker } from 'node:worker_threads';
import { resolve } from 'node:path';
import { config } from '../config.js';
import { db } from '../db/index.js';
import type { MetricsQuery, MetricsData } from '../../metrics.js';
import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js';
class ReportWorker {
private worker: Worker | null = null;
private sequence = 0;
private pending = new Map<number, { resolve: (value: any) => void; reject: (error: Error) => void; timer: NodeJS.Timeout }>();
private flights = new Map<string, Promise<any>>();
async close() {
const worker = this.worker; this.worker = null;
for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Report service stopped')); }
this.pending.clear(); this.flights.clear();
await worker?.terminate();
}
run<T>(kind: string, input: unknown): Promise<T> {
const requestInput = input as { query?: MetricsQuery; now?: number };
const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
const flight = this.flights.get(key);
if (flight) return flight;
if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.'));
db(); // Finish migrations before opening the independent reader.
if (!this.worker) {
const worker = this.worker = new Worker(resolve(process.env.NODE_ENV === 'production' ? 'build/reports/worker.mjs' : '.svelte-kit/reports/worker.mjs'), { workerData: { dbPath: resolve(config.dbPath) } });
worker.on('message', ({ id, value, error, elapsedMs, cached }) => {
const pending = this.pending.get(id); if (!pending) return;
clearTimeout(pending.timer); this.pending.delete(id);
if (process.env.EC_PERF_LOG === 'true') console.info('[report]', { kind, elapsedMs, cached });
if (error) pending.reject(new Error(error)); else pending.resolve(value);
if (!this.pending.size) worker.unref();
});
const fail = (error: Error) => {
if (this.worker !== worker) return;
this.worker = null;
for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(error); }
this.pending.clear(); void worker.terminate();
};
worker.on('error', fail);
worker.on('exit', () => fail(new Error('Report worker stopped. Please retry.')));
}
this.worker.ref();
const id = ++this.sequence;
const promise = new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
// Reset the worker rather than leaving an expensive abandoned query queued.
const worker = this.worker; this.worker = null;
for (const item of this.pending.values()) { clearTimeout(item.timer); item.reject(new Error('Report timed out. Please retry.')); }
this.pending.clear(); void worker?.terminate();
}, 60_000);
this.pending.set(id, { resolve, reject, timer }); this.worker!.postMessage({ id, kind, input });
}).finally(() => { if (this.flights.get(key) === promise) this.flights.delete(key); });
this.flights.set(key, promise);
return promise;
}
}
const metrics = new ReportWorker();
const leaderboard = new ReportWorker();
export const metricsReport = (query: MetricsQuery) => metrics.run<MetricsData>('metrics', { query, now: Date.now() });
export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run<LeaderboardResponse>('leaderboard', query);
export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options });
export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); }
src/lib/server/reports/revisions.ts
+14
View File
@@ -0,0 +1,14 @@
import type Database from 'better-sqlite3';
export function installReportRevisions(handle: Database.Database) {
handle.exec(`CREATE TABLE IF NOT EXISTS report_revisions (topic TEXT PRIMARY KEY, revision INTEGER NOT NULL DEFAULT 0);
INSERT OR IGNORE INTO report_revisions(topic) VALUES ('metrics'), ('leaderboard');`);
for (const [topic, tables] of Object.entries({ metrics: ['metric_samples', 'servers'],
leaderboard: ['leaderboard_matches', 'leaderboard_match_players', 'leaderboard_players'] })) {
for (const table of tables) for (const action of ['INSERT', 'UPDATE', 'DELETE']) {
const condition = table === 'servers' && action === 'UPDATE' ? 'WHEN OLD.destroyed_at IS NOT NEW.destroyed_at OR OLD.name IS NOT NEW.name' : '';
handle.exec(`CREATE TRIGGER IF NOT EXISTS report_${table}_${action} AFTER ${action} ON ${table} ${condition}
BEGIN UPDATE report_revisions SET revision=revision+1 WHERE topic='${topic}'; END;`);
}
}
}
src/lib/server/reports/worker.ts
+33
View File
@@ -0,0 +1,33 @@
import { parentPort } from 'node:worker_threads';
import { db } from '../db/index.js';
import { readMetrics } from '../fleet/metrics.js';
import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js';
import { getPlayerProfile } from '../fleet/player-profile.js';
const cache = new Map<string, { value: unknown; bytes: number }>();
let bytes = 0;
parentPort!.on('message', ({ id, kind, input }) => {
const started = performance.now();
try {
const result = db().transaction(() => {
const topic = kind === 'metrics' ? 'metrics' : 'leaderboard';
const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number };
const now = kind === 'metrics' ? input.now : Date.now();
const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]);
const saved = cache.get(key);
if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; }
const value = kind === 'metrics' ? readMetrics(input.query, now)
: kind === 'leaderboard' ? getLeaderboard(input)
: { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) };
const size = Buffer.byteLength(JSON.stringify(value));
if (size < 16 * 1024 * 1024) {
cache.set(key, { value, bytes: size }); bytes += size;
while (cache.size > 64 || bytes > 32 * 1024 * 1024) {
const oldest = cache.keys().next().value!; bytes -= cache.get(oldest)!.bytes; cache.delete(oldest);
}
}
return { value, cached: false };
})();
parentPort!.postMessage({ id, ...result, elapsedMs: performance.now() - started });
} catch (cause) { parentPort!.postMessage({ id, error: cause instanceof Error ? cause.message : 'Report failed' }); }
});
src/lib/ui/campaigns.ts
+15
View File
@@ -0,0 +1,15 @@
/** Load only the requested history, respecting the API's per-request page bound. */
export async function fetchCampaigns<T>(filters: Record<string, string>, count: number, signal?: AbortSignal): Promise<{ deployments: T[]; total: number }> {
const deployments: T[] = [];
let total = Infinity;
while (deployments.length < Math.min(count, total)) {
const params = new URLSearchParams({ ...filters, limit: String(Math.min(500, count - deployments.length)), offset: String(deployments.length) });
const response = await fetch(`/api/campaigns?${params}`, { signal });
if (!response.ok) throw new Error('Could not refresh campaigns.');
const page = await response.json() as { deployments: T[]; total: number };
total = page.total;
deployments.push(...page.deployments);
if (!page.deployments.length) break;
}
return { deployments, total: Number.isFinite(total) ? total : 0 };
}
src/lib/ui/poll.ts
+12 -5
View File
@@ -1,25 +1,32 @@
/** Poll visible page content without overlapping requests. Fleet watches use their own monitor. */
export function pollVisible(task: () => void | Promise<unknown>, interval: number, immediate = true): () => void {
export function pollVisible(task: (signal: AbortSignal) => unknown | Promise<unknown>, interval: number, immediate = true): () => void {
let stopped = false;
let running = false;
let timer: ReturnType<typeof setTimeout> | undefined;
let controller: AbortController | undefined;
let failures = 0;
const visible = () => document.visibilityState !== 'hidden';
const run = async () => {
if (stopped || running || !visible()) return;
clearTimeout(timer);
running = true;
try { await task(); }
catch (error) { console.error('Page refresh failed', error); }
controller = new AbortController();
const timeout = setTimeout(() => controller?.abort(), 30_000);
try { failures = await task(controller.signal) === false ? failures + 1 : 0; }
catch (error) { if (!controller.signal.aborted) { failures++; console.error('Page refresh failed', error); } }
finally {
clearTimeout(timeout);
controller = undefined;
running = false;
if (!stopped && visible()) timer = setTimeout(run, interval);
if (!stopped && visible()) timer = setTimeout(run, interval * Math.min(8, 2 ** failures));
}
};
const visibility = () => {
clearTimeout(timer);
if (!visible()) controller?.abort();
if (document.visibilityState !== 'hidden') void run();
};
document.addEventListener('visibilitychange', visibility);
if (immediate) void run(); else timer = setTimeout(run, interval);
return () => { stopped = true; clearTimeout(timer); document.removeEventListener('visibilitychange', visibility); };
return () => { stopped = true; clearTimeout(timer); controller?.abort(); document.removeEventListener('visibilitychange', visibility); };
}
src/lib/ui/stores/fleet.ts
+82 -30
View File
@@ -1,7 +1,9 @@
import { freshStatusTimestamp } from '$lib/server-freshness.js';
import { pollVisible } from '../poll.js';
import { selectServerObservation } from '$lib/server-observation.js';
import { derived, writable } from 'svelte/store';
import { derived, writable, get } from 'svelte/store';
import { browser } from '$app/environment';
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
@@ -11,9 +13,30 @@ export const fleetError = writable('');
export const fleetRefreshing = writable(false);
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) =>
snapshot.servers.map(server => {
const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
const times = snapshot.servers.flatMap(server => {
const status = live.get(server.name);
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
});
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
});
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
const now = get(uiNow);
const next = snapshot.servers.map(server => {
const candidate = live.get(server.name);
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
const match = matches.get(server.name);
const report = reports.get(server.name);
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
const saved = observed.get(server.name);
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
const base = status ? { ...server, last_players: status.players, last_map: status.map,
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
@@ -23,38 +46,65 @@ export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, ma
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
telemetry: status.telemetry ?? server.telemetry } : server;
const current = selectServerObservation(base, reports.get(server.name), now);
const match = matches.get(server.name);
// Apply lifecycle events immediately, until a newer status observes them
// or confirms that the lobby has emptied.
if (!match || current.health !== 'online'
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|| match.receivedAt < (current.game_started_at ?? '')
|| match.receivedAt < (current.game_ended_at ?? '')) return current;
return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
game_ended_at: match.event === 'game_end' ? match.receivedAt : null };
}));
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
});
const names = new Set(snapshot.servers.map(server => server.name));
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
return next;
});
let snapshotServerNow = 0;
export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
if (!browser || snapshot.serverNow < snapshotServerNow) return;
snapshotServerNow = snapshot.serverNow;
syncServerTime(snapshot.serverNow);
fleetIntegrations.set(snapshot.integrations);
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
}
function getIntegrations() { return get(fleetIntegrations); }
let inFlight: Promise<void> | null = null;
export function refreshFleet(): Promise<void> {
let refreshRequested = false;
export function refreshFleet(afterCurrent = false): Promise<void> {
if (!browser) return Promise.resolve();
if (inFlight) return inFlight;
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
inFlight = (async () => {
fleetRefreshing.set(true);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 12_000);
const requestedAt = Date.now();
try {
const responses = await Promise.all([
fetch('/api/servers', { signal: controller.signal }),
fetch('/api/fleet', { signal: controller.signal })
const results = await Promise.allSettled([
(async () => {
const response = await fetch('/api/servers', { signal: controller.signal });
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
const servers = await response.json();
if (servers.serverNow < snapshotServerNow) return;
snapshotServerNow = servers.serverNow;
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
})(),
(async () => {
const response = await fetch('/api/fleet', { signal: controller.signal });
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
const operations = await response.json();
fleet.update(value => ({ ...value, operations: operations.operations }));
})()
]);
if (responses.some(response => !response.ok)) throw new Error('Could not refresh Fleet. Retrying automatically.');
const [servers, operations] = await Promise.all(responses.map(response => response.json()));
syncServerTime(servers.serverNow);
const failure = results.find(result => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
fleetError.set('');
fleetIntegrations.set(servers.integrations);
fleet.set({ servers: servers.servers, operations: operations.operations, receivedAt: requestedAt });
} catch (error) {
fleetError.set(error instanceof Error && error.name !== 'AbortError'
? error.message : 'Fleet refresh timed out. Retrying automatically.');
@@ -62,23 +112,25 @@ export function refreshFleet(): Promise<void> {
clearTimeout(timeout);
fleetRefreshing.set(false);
}
})().finally(() => { inFlight = null; });
})().finally(() => {
inFlight = null;
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
});
return inFlight;
}
let stop: (() => void) | null = null;
export function startFleetMonitor(): () => void {
if (!browser || stop) return () => {};
void refreshFleet();
const interval = setInterval(() => {
if (document.visibilityState !== 'hidden') void refreshFleet();
}, 10_000);
const onFocus = () => { if (document.visibilityState === 'visible') void refreshFleet(); };
document.addEventListener('visibilitychange', onFocus);
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
const changed = () => {
if (refreshTimer || document.visibilityState === 'hidden') return;
refreshTimer = setTimeout(() => { refreshTimer = undefined; void refreshFleet(true); }, 250);
};
window.addEventListener('ec:fleet-changed', changed);
stop = () => {
clearInterval(interval);
document.removeEventListener('visibilitychange', onFocus);
stop = null;
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
};
return stop;
}
src/lib/ui/stores/pipeline.ts
+3
View File
@@ -26,6 +26,9 @@ function updateDeployment(id: string, mutate: (d: PipelineDeployment) => void):
export function startPipelineStream(): (() => void) | undefined {
if (!browser || source) return;
source = new EventSource('/api/events/pipeline');
for (const event of ['server-changed', 'server-state', 'deployment-opened', 'fleet-invalidated']) {
source.addEventListener(event, () => window.dispatchEvent(new Event('ec:fleet-changed')));
}
source.addEventListener('deployment-opened', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { deploymentId: string; action: string; servers: string[] };
src/lib/ui/stores/servers.ts
+6 -5
View File
@@ -17,6 +17,7 @@ export interface RconEndpoint {
}
export interface ServerStatus {
receivedAt?: number;
server: string;
containerState?: string | null;
containerCheckedAt?: string | null;
@@ -91,17 +92,17 @@ export function startServerStream(): (() => void) | undefined {
});
});
source.addEventListener('server-status', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number };
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number; delta?: boolean };
syncServerTime(data.serverNow);
statusByServer.update(() => {
const m = new Map<string, ServerStatus>();
for (const e of data.entries) m.set(e.server, e);
statusByServer.update(previous => {
const m = data.delta ? new Map(previous) : new Map<string, ServerStatus>();
for (const e of data.entries) m.set(e.server, { ...e, receivedAt: Date.now() });
return m;
});
statusReceivedAt.set(Date.now());
});
if (tickInterval === null) {
tickInterval = setInterval(() => _uiNow.set(serverTimeNow()), 1000);
tickInterval = setInterval(() => { if (typeof document === 'undefined' || document.visibilityState !== 'hidden') _uiNow.set(serverTimeNow()); }, 1000);
}
return () => { source?.close(); source = null; if (tickInterval) clearInterval(tickInterval); tickInterval = null; };
}