Improve application responsiveness and background processing

This commit is contained in:
2026-10-04 19:56:06 -04:00
parent 1392a082da
commit ed004ddbcf
70 changed files with 1324 additions and 345 deletions
+12 -1
View File
@@ -3,6 +3,8 @@ import { migrate } from '$lib/server/db/migrate.js';
import { startWorker } from '$lib/server/pipeline/worker.js';
import { interruptDeployments } from '$lib/server/provisioning/history.js';
import { authenticationHandle } from '$lib/server/auth/guard.js';
import { emit } from '$lib/server/pipeline/bus.js';
import { startPerformanceMonitor } from '$lib/server/performance.js';
let booted = false;
function boot(): void {
@@ -11,6 +13,7 @@ function boot(): void {
migrate();
interruptDeployments();
startWorker();
startPerformanceMonitor();
}
// SvelteKit calls handle() for every request; using the first request as the
@@ -18,5 +21,13 @@ function boot(): void {
// out-of-band init hook.
export const handle: Handle = async (input) => {
boot();
return authenticationHandle(input);
const started = performance.now();
const response = await authenticationHandle(input);
const elapsed = performance.now() - started;
response.headers.append('Server-Timing', `app;dur=${elapsed.toFixed(1)}`);
if (process.env.EC_PERF_LOG === 'true' && elapsed > 250 && !input.event.route.id?.startsWith('/api/agent/')) {
console.info('[request]', { route: input.event.route.id, method: input.event.request.method, status: response.status, elapsedMs: Math.round(elapsed) });
}
if (input.event.locals.user && !['GET','HEAD','OPTIONS'].includes(input.event.request.method) && response.status < 400) emit({ type: 'fleet-invalidated' });
return response;
};
+2
View File
@@ -1,4 +1,5 @@
import Database from 'better-sqlite3';
import { installReportRevisions } from '../reports/revisions.js';
import { mkdirSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { config } from '../config.js';
@@ -368,6 +369,7 @@ export function db(): Database.Database {
backfillLeaderboard(handle);
initializeFleetSettings(handle);
migrateNativeInstances(handle);
installReportRevisions(handle);
} catch (err) {
handle.close();
throw err;
+33 -24
View File
@@ -10,6 +10,11 @@ const metrics = new Set<string>(leaderboardMetrics.map(metric => metric.key));
export const MAX_LEADERBOARD_OFFSET = 1_000_000;
type RankedEntry = Omit<LeaderboardEntry, 'records'>;
interface RankedScope { revision: number; entries: RankedEntry[]; maps: LeaderboardMap[]; summary: LeaderboardResponse['summary'] }
const scopeCaches = new WeakMap<object, Map<string, RankedScope>>();
// Match SQLite's built-in lower(), which folds ASCII rather than Unicode.
const searchText = (value: string) => value.replace(/[A-Z]/g, ch => ch.toLowerCase());
interface LeaderboardRow extends Omit<RankedEntry, 'guid'> {
// A left join keeps metadata available when the requested page has no players.
guid: string | null;
@@ -91,21 +96,15 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
SELECT measures.*, p.name, RANK() OVER (ORDER BY measures.${metric} DESC) AS rank
FROM measures JOIN identities p ON p.guid = measures.guid AND p.recency = 1 ${qualification}
)`;
if (exactGuid !== undefined) {
const entries = handle.prepare(`${aggregated} SELECT * FROM ranked WHERE guid = ?`)
.all(...parameters, exactGuid) as RankedEntry[];
return addRecordSources(handle, entries, options)[0] ?? null;
}
const query = options.q?.trim() ?? '';
const search = query ? 'WHERE instr(lower(name), lower(?)) > 0 OR instr(lower(guid), lower(?)) > 0' : '';
// Reuse one ranked set for the page and matching count. Summaries deliberately
// cover the full game/map scope, including players below an efficiency threshold.
const rows = handle.prepare(`${aggregated}, searched AS MATERIALIZED (
SELECT * FROM ranked ${search}
), page AS (
SELECT * FROM searched ORDER BY ${metric} DESC, name COLLATE NOCASE, name, guid LIMIT ? OFFSET ?
), playerSummary AS (
let revision = -1;
try { revision = (handle.prepare("SELECT revision FROM report_revisions WHERE topic='leaderboard'").get() as { revision: number }).revision; }
catch { /* Standalone history tools can operate without cache metadata. */ }
let cache = scopeCaches.get(handle);
if (!cache) { cache = new Map(); scopeCaches.set(handle, cache); }
const cacheKey = JSON.stringify([options.game ?? '', options.map ?? '', metric]);
let saved = cache.get(cacheKey);
if (!saved || revision < 0 || saved.revision !== revision) {
const rows = handle.prepare(`${aggregated}, playerSummary AS (
SELECT COUNT(*) AS players, COALESCE(SUM(kills), 0) AS kills,
COALESCE(SUM(roundsSurvived), 0) AS roundsSurvived,
COALESCE(MAX(highestRound), 0) AS highestRound FROM totals
@@ -113,21 +112,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
SELECT COUNT(DISTINCT match_id) AS matches, MIN(matchStartedAt) AS firstTrackedAt,
MAX(matchUpdatedAt) AS updatedAt FROM observations
), metadata AS (
SELECT (SELECT COUNT(*) FROM searched) AS total, playerSummary.*, matchSummary.*
FROM playerSummary CROSS JOIN matchSummary
) SELECT page.*, metadata.total AS matchingTotal, metadata.players AS summaryPlayers,
SELECT playerSummary.*, matchSummary.* FROM playerSummary CROSS JOIN matchSummary
) SELECT ranked.*, metadata.players AS summaryPlayers,
metadata.matches AS summaryMatches, metadata.kills AS summaryKills,
metadata.roundsSurvived AS summaryRounds, metadata.highestRound AS summaryHighestRound,
metadata.firstTrackedAt AS summaryFirstTrackedAt, metadata.updatedAt AS summaryUpdatedAt
FROM metadata LEFT JOIN page ON 1
ORDER BY page.${metric} DESC, page.name COLLATE NOCASE, page.name, page.guid`)
.all(...parameters, ...(query ? [query, query] : []), limit, offset) as LeaderboardRow[];
FROM metadata LEFT JOIN ranked ON 1
ORDER BY ranked.${metric} DESC, ranked.name COLLATE NOCASE, ranked.name, ranked.guid`)
.all(...parameters) as LeaderboardRow[];
const rankedEntries = rows.filter((row): row is LeaderboardRow & { guid: string } => row.guid !== null)
.map(({ matchingTotal, summaryPlayers, summaryMatches, summaryKills, summaryRounds,
summaryHighestRound, summaryFirstTrackedAt, summaryUpdatedAt, ...entry }) => entry);
const entries = addRecordSources(handle, rankedEntries, options);
const metadata = rows[0];
const total = metadata.matchingTotal;
const summary: LeaderboardResponse['summary'] = {
players: metadata.summaryPlayers, matches: metadata.summaryMatches, kills: metadata.summaryKills,
roundsSurvived: metadata.summaryRounds, highestRound: metadata.summaryHighestRound,
@@ -140,5 +136,18 @@ function readLeaderboard(options: LeaderboardOptions, exactGuid?: string): Leade
GROUP BY game, map_key HAVING id = MAX(id)
ORDER BY game, name COLLATE NOCASE, map_key`)
.all(...(options.game ? [options.game] : [])) as LeaderboardMap[];
return { entries, maps, total, limit, offset, metric, summary };
saved = { revision, entries: rankedEntries, maps, summary };
cache.delete(cacheKey);
if (rankedEntries.length <= 100_000) cache.set(cacheKey, saved);
// Bound total retained players as well as the number of scopes.
while (cache.size > 8 || [...cache.values()].reduce((n, scope) => n + scope.entries.length, 0) > 100_000) cache.delete(cache.keys().next().value!);
}
if (exactGuid !== undefined) {
const entry = saved.entries.find(entry => entry.guid === exactGuid);
return entry ? addRecordSources(handle, [entry], options)[0] : null;
}
const query = searchText(options.q?.trim() ?? '');
const matching = query ? saved.entries.filter(entry => searchText(entry.name).includes(query) || searchText(entry.guid).includes(query)) : saved.entries;
const entries = addRecordSources(handle, matching.slice(offset, offset + limit), options);
return { entries, maps: saved.maps, total: matching.length, limit, offset, metric, summary: saved.summary };
}
+10 -2
View File
@@ -179,6 +179,10 @@ export function setServerField(
field: keyof ServerRow,
value: string | number | null
): void {
setServerFields(name, { [field]: value });
}
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
const allowed: Array<keyof ServerRow> = [
'komodo_type',
'komodo_name',
@@ -211,8 +215,12 @@ export function setServerField(
'game_ended_at',
'game_cycle_key'
];
if (!allowed.includes(field)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${field} = ? WHERE name = ?`).run(value, name);
const entries = Object.entries(fields);
if (!entries.length) return;
for (const [field] of entries) if (!allowed.includes(field as keyof ServerRow)) throw new Error(`refusing to set ${field}`);
db().prepare(`UPDATE servers SET ${entries.map(([field]) => `${field} = ?`).join(', ')} WHERE name = ?
AND (${entries.map(([field]) => `${field} IS NOT ?`).join(' OR ')})`)
.run(...entries.map(([,value]) => value), name, ...entries.map(([,value]) => value));
}
/**
+51
View File
@@ -0,0 +1,51 @@
import { db } from '../db/index.js';
import { listServers } from './servers.js';
import { observationReader } from './status.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { runningVersions } from '../integrations/plutonium.js';
import { listHosts } from '../hosts/index.js';
import type { NativeInstance } from '../backends/native.js';
export function fleetSnapshot() {
const rconHost = config.rcon.host;
const hosts = listHosts();
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
const versions = new Map(runningVersions().map(version => [version.server, version]));
const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
const holds = new Map<string, { target: string; betaId: number }[]>();
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
{ server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
}
const observe = observationReader();
const servers = listServers().map((s) => {
const instance = s.native_instance_id ? instances.get(s.name) : null;
return {
...observe(s),
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: holds.get(s.name) ?? []
};
});
return {
servers,
serverNow: Date.now(),
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
komodo: komodoConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
defaultRconHost: rconHost
}
};
}
+41 -5
View File
@@ -4,21 +4,57 @@ import { isProvisioning } from './provisioning.js';
import { restartSettling } from '../provisioning/restarts.js';
import { type ServerRow } from './servers.js';
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { restartRequestInFlight } from '../provisioning/restarts.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
/** One set of evidence reads per snapshot, rather than four queries per server. */
export function observationReader() {
const handle = db();
const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
{ server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM (
SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
{ server: string; at: string | null }[]).map(row => [row.server, row.at]));
const reports = new Map<string, GameTelemetry>();
for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
{ server: string; received_at: string; payload: string }[]) {
try {
const saved = JSON.parse(row.payload) as GameTelemetry;
const event = parseGameEvent(saved);
if (event.server === row.server) reports.set(row.server, { ...event, receivedAt: row.received_at, timing: saved.timing });
} catch { /* Invalid retained telemetry is unavailable. */ }
}
return (server: ServerRow) => {
const operation = operations.get(server.name), restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
});
};
}
/** Shared snapshot for REST, status events, metrics, and other server-side readers. */
export function serverObservation(server: ServerRow) {
export function serverObservation(server: ServerRow, evidence?: Evidence) {
let roster = null;
try { roster = server.last_roster ? JSON.parse(server.last_roster) : null; } catch { /* unavailable */ }
const rcon: GameObservation = { last_players: server.last_players, last_map: server.last_map,
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
const provisioning = isProvisioning(server.name);
const state = restartSettling(server.name) ? 'restarting' : server.container_state;
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
const ready = !provisioning && (!state || state === 'running');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
telemetry: telemetryReport(server.name) ?? undefined,
telemetry_after: latestGameSubmission(server.name),
telemetry: (evidence ? evidence.telemetry : telemetryReport(server.name)) ?? undefined,
telemetry_after: evidence ? evidence.submission : latestGameSubmission(server.name),
last_error: ready ? server.last_error : null,
replySnippet: ready ? server.last_reply_snippet : null,
active_since: ready ? server.active_since : null, inactive_since: ready ? server.inactive_since : null,
+16 -9
View File
@@ -18,14 +18,14 @@ type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id,
...(row.bytes_verified == null ? {} : { bytesVerified: row.bytes_verified }),
...(row.transfer_progress ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } : {}) });
function versionFromRow(row: VersionRow): GameLibraryVersion {
const files = JSON.parse(row.manifest) as GameFileEntry[];
function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches?: Map<string, HostLibraryCache[]>): GameLibraryVersion {
const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
description: row.description, state: row.state, sizeBytes: row.size_bytes,
files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
caches: summaryCaches ? summaryCaches.get(row.id) ?? [] : (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
}
export function getLibraryVersion(id: string): GameLibraryVersion | null {
const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
@@ -52,12 +52,19 @@ export function latestPublishedLibraries() {
return version ? [{ ...version, name: game.name }] : [];
});
}
export function listLibraries(): GameLibrary[] {
export function listLibraries(summary = false): GameLibrary[] {
reconcileLibraryOperations();
let summaryCaches: Map<string, HostLibraryCache[]> | undefined;
if (summary) {
summaryCaches = new Map();
for (const row of database().prepare('SELECT * FROM host_library_caches').all() as CacheRow[]) {
const items = summaryCaches.get(row.version_id) ?? []; items.push(cacheFromRow(row)); summaryCaches.set(row.version_id, items);
}
}
const latest = new Map(latestPublishedLibraries().map(version => [version.game, version.id]));
return GAME_FILE_GAMES.map(game => {
const versions = (database().prepare(`SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(versionFromRow);
const versions = (database().prepare(`SELECT ${summary ? `v.id, v.library_id, v.version, v.description, v.dlc, v.state, v.size_bytes, v.manifest_sha256, v.created_at, v.published_at, v.storage_location, v.source_host_id, v.source_volume, v.error, v.operation_id, CASE WHEN json_array_length(v.manifest)>0 THEN json_array_length(v.manifest) ELSE (SELECT COUNT(*) FROM game_file_upload_entries e WHERE e.version_id=v.id) END AS file_count` : 'v.*'}, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(row => versionFromRow(row, summaryCaches));
const firstLibrary = database().prepare('SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game.id) as { created_at: string } | undefined;
return { id: game.id, name: game.name, game: game.id, description: '', createdAt: firstLibrary?.created_at ?? '', latestVersionId: latest.get(game.id) ?? null, versions };
});
@@ -392,9 +399,9 @@ export function reconcileLibraryOperations(): void {
/** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */
export function libraryOverview() {
const hosts = listHosts();
return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
return { libraries: listLibraries(true).map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
...version, caches: hosts.map(host => {
const cached = getHostLibraryCache(host.id, version.id);
const cached = version.caches.find(cache => cache.hostId === host.id);
if (cached) return { ...cached, status: cached.status === 'ready' && library.latestVersionId !== version.id ? 'outdated' as const : cached.status };
return {
hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,
+35 -5
View File
@@ -44,20 +44,50 @@ export function validateUploadPlan(input: unknown): UploadFilePlan[] {
return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
}
const uploadLocks = new Map<string, Promise<void>>();
interface UploadLocks { barrier: Promise<void>; writers: number; files: Map<string, Promise<void>>; active: Set<Promise<void>> }
const uploadLocks = new Map<string, UploadLocks>();
function locksFor(key: string): UploadLocks {
let locks = uploadLocks.get(key);
if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); }
return locks;
}
function releaseLocks(key: string, locks: UploadLocks) {
if (!locks.writers && !locks.active.size && uploadLocks.get(key) === locks) uploadLocks.delete(key);
}
/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
const key = resolve(root);
const previous = uploadLocks.get(key) ?? Promise.resolve();
const locks = locksFor(key);
// Capture readers admitted before publication, then prevent later readers
// from starting until publication finishes (including failed publication).
const previous = Promise.all([locks.barrier, ...locks.active]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
uploadLocks.set(key, current);
locks.barrier = current;
locks.writers++;
await previous;
try { return await callback(); }
finally {
release();
if (uploadLocks.get(key) === current) uploadLocks.delete(key);
locks.writers--;
releaseLocks(key, locks);
}
}
/** Different files may transfer together; offsets for the same file remain serial. */
export async function withUploadFileLock<T>(root: string, file: string, callback: () => Promise<T>): Promise<T> {
const key = resolve(root), locks = locksFor(key);
const previous = Promise.all([locks.barrier, locks.files.get(file)]);
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
locks.files.set(file, current); locks.active.add(current);
await previous;
try { return await callback(); }
finally {
release(); locks.active.delete(current);
if (locks.files.get(file) === current) locks.files.delete(file);
releaseLocks(key, locks);
}
}
@@ -95,7 +125,7 @@ export async function receiveUploadChunk(
}
if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
return withUploadLock(root, async () => {
return withUploadFileLock(root, file.path, async () => {
await beforeWrite?.();
const progress = await getUploadFileProgress(root, file);
if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
+25 -5
View File
@@ -1,4 +1,5 @@
import { recordAgentReceipt } from '../provisioning/agent-events.js';
import { EventEmitter } from 'node:events';
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
import { isIP } from 'node:net';
import { db } from '../db/index.js';
@@ -8,6 +9,16 @@ import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, Hos
export const AGENT_PROTOCOL_VERSION = 1;
export const HEARTBEAT_MAX_AGE_MS = 90_000;
const now = () => new Date().toISOString();
const workEvents = new EventEmitter();
workEvents.setMaxListeners(0);
export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise<void> {
return new Promise(resolve => {
const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); };
const timer = setTimeout(done, timeoutMs);
workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true });
if (signal.aborted) done();
});
}
const digest = (token: string) => createHash('sha256').update(token).digest('hex');
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
@@ -113,6 +124,7 @@ export function deleteHost(id: string): { deleted: boolean; operationId?: string
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
.run(id, operationId, digest(token), timestamp, timestamp);
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
queueMicrotask(() => workEvents.emit(id));
return { deleted: false, operationId };
}).immediate();
}
@@ -287,7 +299,7 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
@@ -300,7 +312,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
return database().transaction(() => {
const operation = database().transaction(() => {
if (options.idempotencyKey) {
const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
if (existing) {
@@ -315,8 +327,10 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
return getHostOperation(id)!;
})();
queueMicrotask(() => workEvents.emit(hostId));
return operation;
}
export function pendingHostOperations(hostId: string): HostOperation[] {
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
@@ -328,8 +342,12 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
return database().transaction(() => {
const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
@@ -342,7 +360,7 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
})();
}
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
return database().transaction(() => {
const operation = database().transaction(() => {
requireHost(hostId);
const operation = getHostOperation(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
@@ -367,6 +385,8 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
}
return getHostOperation(id)!;
})();
queueMicrotask(() => workEvents.emit(`operation:${id}`));
return operation;
}
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
const deadline = Date.now() + timeoutMs;
@@ -375,7 +395,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
if (!operation) throw new Error('Host operation was not found.');
if (['completed','failed','unknown'].includes(operation.state)) return operation;
if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
await new Promise(resolve => setTimeout(resolve, 200));
await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
}
}
+8 -3
View File
@@ -22,13 +22,18 @@ export function freshRelease(): number | null {
export function runningVersions(): PlutoniumVersion[] {
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
const target = freshRelease();
const byServer = new Map(versions.map(version => [version.server, version]));
const epochs = new Map((db().prepare(`SELECT server, MAX(at) AS at FROM (
SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as { server: string; at: string | null }[]).map(row => [row.server, row.at]));
const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
return listServers().map(server => {
const value = versions.find(v => v.server === server.name)
const value = byServer.get(server.name)
?? { server: server.name, revision: null, checked_at: null, error: null };
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
const epoch = latestGameSubmission(server.name);
const epoch = epochs.get(server.name);
const stopped = server.desired_state === 'stopped';
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !restartRecoveryPending(server.name);
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name);
const revision = valid && !stopped ? value.revision : null;
return { ...value, revision, stopped,
update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'
+13
View File
@@ -0,0 +1,13 @@
import { monitorEventLoopDelay } from 'node:perf_hooks';
let started = false;
export function startPerformanceMonitor() {
if (started || process.env.EC_PERF_LOG !== 'true') return;
started = true;
const delay = monitorEventLoopDelay({ resolution: 20 }); delay.enable();
setInterval(() => {
console.info('[performance]', { eventLoopP95Ms: Math.round(delay.percentile(95) / 1e6),
eventLoopMaxMs: Math.round(delay.max / 1e6), rssMiB: Math.round(process.memoryUsage().rss / 1048576) });
delay.reset();
}, 60_000).unref();
}
+10 -1
View File
@@ -46,6 +46,7 @@ export type BetaStage =
export type BetaState = 'running' | 'ready-promote' | 'promoted' | 'rolled-back';
export type PipelineEvent =
| { type: 'fleet-invalidated' }
| { type: 'server-telemetry'; entries: GameTelemetry[] }
| { type: 'deployment-opened'; deploymentId: string; action: string; servers: string[] }
| { type: 'server-changed'; deploymentId: string; server: string; files: number }
@@ -58,7 +59,7 @@ export type PipelineEvent =
error: string | null;
}
| { type: 'sweep-tick'; frozen: boolean; reason?: string }
| { type: 'server-status'; entries: ServerStatusEntry[] }
| { type: 'server-status'; entries: ServerStatusEntry[]; delta?: boolean }
| { type: 'beta-opened'; betaId: number; scriptIds: number[]; servers: string[] }
| {
type: 'beta-server-stage';
@@ -78,9 +79,17 @@ class Bus extends EventEmitter {
}
const bus = new Bus();
let invalidationPending = false;
export function invalidateFleet(): void {
if (invalidationPending) return;
invalidationPending = true;
// Flush after nested database transactions have committed.
queueMicrotask(() => { invalidationPending = false; bus.emit('event', { type: 'fleet-invalidated' }); });
}
export function emit(event: PipelineEvent): void {
bus.emit('event', event);
if (event.type === 'beta-opened' || event.type === 'beta-state' || event.type === 'beta-server-stage') invalidateFleet();
}
export function subscribe(listener: (event: PipelineEvent) => void): () => void {
+44 -29
View File
@@ -6,10 +6,10 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation } from '../fleet/status.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
@@ -76,6 +76,15 @@ function encodeError(reason: RconFailure, detail: string): string {
const pendingStatus = new Map<string, Promise<number | null>>();
const statusQueue: Array<() => Promise<void>> = [];
let activeStatus = 0;
const changedStatus = new Map<string, ServerStatusEntry>();
function publishStatus(entry: ServerStatusEntry) {
const scheduled = changedStatus.size > 0;
changedStatus.set(entry.server, entry);
if (!scheduled) queueMicrotask(() => {
const entries = [...changedStatus.values()]; changedStatus.clear();
emit({ type: 'server-status', delta: true, entries });
});
}
function drainStatusQueue(): void {
while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
@@ -95,9 +104,9 @@ function queueStatus(server: string): Promise<number | null> {
let capacity: number | null = null;
try {
capacity = await refreshServerStatus(server);
// Consumers expect a full snapshot. Re-read it now so neither game
// data nor container evidence waits for the slowest fleet member.
emit({ type: 'server-status', entries: listServers().map(statusEntry) });
// Publish this server's latest evidence without waiting for the fleet.
const current = getServer(server);
if (current) publishStatus(statusEntry(current));
} catch (err) { console.error(`[worker] status error for ${server}:`, err); }
finally { pendingStatus.delete(server); finish(capacity); }
});
@@ -113,8 +122,9 @@ export async function fanoutStatus(): Promise<void> {
try {
// Metrics remain complete fleet collections, using freshness at capture
// time rather than the status saved at the beginning of a long scan.
const observe = observationReader();
recordMetrics(listServers().map(srv => {
const current = serverObservation(srv);
const current = observe(srv);
return { server: srv.name, game: srv.game ?? 'unknown',
map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
players: current.health === 'online' ? current.last_players : null,
@@ -127,6 +137,8 @@ async function refreshServerStatus(server: string): Promise<number | null> {
const srv = getServer(server);
if (!srv) return null;
let capacity: number | null = null;
let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const epoch = latestGameSubmission(srv.name);
const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
@@ -136,53 +148,56 @@ async function refreshServerStatus(server: string): Promise<number | null> {
: await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
// A lifecycle request may be submitted while an inspection is in flight.
if (unchanged()) {
setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
setServerField(srv.name, 'container_checked_at', new Date().toISOString());
updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
updates.container_checked_at = new Date().toISOString();
if (!container.ok) {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
updates.last_error = null;
updates.last_reply_snippet = null;
} else if (!telemetryEvidence(srv.name)) {
flush();
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
if (unchanged()) {
const now = new Date().toISOString();
setServerField(srv.name, 'last_checked_at', now);
setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
updates.last_checked_at = now;
updates.last_reply_snippet = result.rawReply ?? null;
if (result.ok) {
capacity = result.status.capacity ?? null;
setServerField(srv.name, 'last_players', result.status.players);
setServerField(srv.name, 'last_map', result.status.map);
setServerField(srv.name, 'last_gametype', result.status.gametype);
setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
setServerField(srv.name, 'last_seen_at', now);
setServerField(srv.name, 'last_error', null);
} else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
updates.last_players = result.status.players;
updates.last_map = result.status.map;
updates.last_gametype = result.status.gametype;
updates.last_roster = JSON.stringify(result.status.roster);
updates.last_seen_at = now;
updates.last_error = null;
} else updates.last_error = encodeError(result.reason, result.detail);
}
}
}
} else {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
updates.last_error = null;
updates.last_reply_snippet = null;
}
flush();
const current = getServer(srv.name) ?? srv;
const status = serverObservation(current);
// Occupancy timers and match clearing follow the same source as the UI.
if (status.health === 'online' && status.last_players !== null) {
if (status.last_players > 0) {
setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
setServerField(srv.name, 'inactive_since', null);
updates.active_since = current.active_since ?? status.last_seen_at;
updates.inactive_since = null;
} else {
setServerField(srv.name, 'active_since', null);
setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
setServerField(srv.name, 'game_started_at', null);
setServerField(srv.name, 'game_ended_at', null);
updates.active_since = null;
updates.inactive_since = current.inactive_since ?? status.last_seen_at;
updates.game_started_at = null;
updates.game_ended_at = null;
}
}
flush();
return capacity;
}
function statusEntry(srv: ServerRow): ServerStatusEntry {
const final = serverObservation(srv);
export function statusEntry(srv: ServerRow, observe = serverObservation): ServerStatusEntry {
const final = observe(srv);
return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
lastError: final.last_error, replySnippet: final.replySnippet,
+2
View File
@@ -1,4 +1,5 @@
import { db } from '../db/index.js';
import { invalidateFleet } from '../pipeline/bus.js';
import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck,
type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js';
@@ -116,5 +117,6 @@ export function recordProgress(id: string, update: ProgressUpdate): void {
const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json)
VALUES (?, ?, ?, ?, ?, ?)`);
for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null);
if (events.length) invalidateFleet();
})();
}
+5
View File
@@ -0,0 +1,5 @@
import Database from 'better-sqlite3';
import { workerData } from 'node:worker_threads';
const handle = new Database(workerData.dbPath, { readonly: true, fileMustExist: true });
export function db() { return handle; }
+64
View File
@@ -0,0 +1,64 @@
import { Worker } from 'node:worker_threads';
import { resolve } from 'node:path';
import { config } from '../config.js';
import { db } from '../db/index.js';
import type { MetricsQuery, MetricsData } from '../../metrics.js';
import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js';
class ReportWorker {
private worker: Worker | null = null;
private sequence = 0;
private pending = new Map<number, { resolve: (value: any) => void; reject: (error: Error) => void; timer: NodeJS.Timeout }>();
private flights = new Map<string, Promise<any>>();
async close() {
const worker = this.worker; this.worker = null;
for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(new Error('Report service stopped')); }
this.pending.clear(); this.flights.clear();
await worker?.terminate();
}
run<T>(kind: string, input: unknown): Promise<T> {
const requestInput = input as { query?: MetricsQuery; now?: number };
const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
const flight = this.flights.get(key);
if (flight) return flight;
if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.'));
db(); // Finish migrations before opening the independent reader.
if (!this.worker) {
const worker = this.worker = new Worker(resolve(process.env.NODE_ENV === 'production' ? 'build/reports/worker.mjs' : '.svelte-kit/reports/worker.mjs'), { workerData: { dbPath: resolve(config.dbPath) } });
worker.on('message', ({ id, value, error, elapsedMs, cached }) => {
const pending = this.pending.get(id); if (!pending) return;
clearTimeout(pending.timer); this.pending.delete(id);
if (process.env.EC_PERF_LOG === 'true') console.info('[report]', { kind, elapsedMs, cached });
if (error) pending.reject(new Error(error)); else pending.resolve(value);
if (!this.pending.size) worker.unref();
});
const fail = (error: Error) => {
if (this.worker !== worker) return;
this.worker = null;
for (const pending of this.pending.values()) { clearTimeout(pending.timer); pending.reject(error); }
this.pending.clear(); void worker.terminate();
};
worker.on('error', fail);
worker.on('exit', () => fail(new Error('Report worker stopped. Please retry.')));
}
this.worker.ref();
const id = ++this.sequence;
const promise = new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
// Reset the worker rather than leaving an expensive abandoned query queued.
const worker = this.worker; this.worker = null;
for (const item of this.pending.values()) { clearTimeout(item.timer); item.reject(new Error('Report timed out. Please retry.')); }
this.pending.clear(); void worker?.terminate();
}, 60_000);
this.pending.set(id, { resolve, reject, timer }); this.worker!.postMessage({ id, kind, input });
}).finally(() => { if (this.flights.get(key) === promise) this.flights.delete(key); });
this.flights.set(key, promise);
return promise;
}
}
const metrics = new ReportWorker();
const leaderboard = new ReportWorker();
export const metricsReport = (query: MetricsQuery) => metrics.run<MetricsData>('metrics', { query, now: Date.now() });
export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run<LeaderboardResponse>('leaderboard', query);
export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options });
export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); }
+14
View File
@@ -0,0 +1,14 @@
import type Database from 'better-sqlite3';
export function installReportRevisions(handle: Database.Database) {
handle.exec(`CREATE TABLE IF NOT EXISTS report_revisions (topic TEXT PRIMARY KEY, revision INTEGER NOT NULL DEFAULT 0);
INSERT OR IGNORE INTO report_revisions(topic) VALUES ('metrics'), ('leaderboard');`);
for (const [topic, tables] of Object.entries({ metrics: ['metric_samples', 'servers'],
leaderboard: ['leaderboard_matches', 'leaderboard_match_players', 'leaderboard_players'] })) {
for (const table of tables) for (const action of ['INSERT', 'UPDATE', 'DELETE']) {
const condition = table === 'servers' && action === 'UPDATE' ? 'WHEN OLD.destroyed_at IS NOT NEW.destroyed_at OR OLD.name IS NOT NEW.name' : '';
handle.exec(`CREATE TRIGGER IF NOT EXISTS report_${table}_${action} AFTER ${action} ON ${table} ${condition}
BEGIN UPDATE report_revisions SET revision=revision+1 WHERE topic='${topic}'; END;`);
}
}
}
+33
View File
@@ -0,0 +1,33 @@
import { parentPort } from 'node:worker_threads';
import { db } from '../db/index.js';
import { readMetrics } from '../fleet/metrics.js';
import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js';
import { getPlayerProfile } from '../fleet/player-profile.js';
const cache = new Map<string, { value: unknown; bytes: number }>();
let bytes = 0;
parentPort!.on('message', ({ id, kind, input }) => {
const started = performance.now();
try {
const result = db().transaction(() => {
const topic = kind === 'metrics' ? 'metrics' : 'leaderboard';
const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number };
const now = kind === 'metrics' ? input.now : Date.now();
const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]);
const saved = cache.get(key);
if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; }
const value = kind === 'metrics' ? readMetrics(input.query, now)
: kind === 'leaderboard' ? getLeaderboard(input)
: { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) };
const size = Buffer.byteLength(JSON.stringify(value));
if (size < 16 * 1024 * 1024) {
cache.set(key, { value, bytes: size }); bytes += size;
while (cache.size > 64 || bytes > 32 * 1024 * 1024) {
const oldest = cache.keys().next().value!; bytes -= cache.get(oldest)!.bytes; cache.delete(oldest);
}
}
return { value, cached: false };
})();
parentPort!.postMessage({ id, ...result, elapsedMs: performance.now() - started });
} catch (cause) { parentPort!.postMessage({ id, error: cause instanceof Error ? cause.message : 'Report failed' }); }
});
+15
View File
@@ -0,0 +1,15 @@
/** Load only the requested history, respecting the API's per-request page bound. */
export async function fetchCampaigns<T>(filters: Record<string, string>, count: number, signal?: AbortSignal): Promise<{ deployments: T[]; total: number }> {
const deployments: T[] = [];
let total = Infinity;
while (deployments.length < Math.min(count, total)) {
const params = new URLSearchParams({ ...filters, limit: String(Math.min(500, count - deployments.length)), offset: String(deployments.length) });
const response = await fetch(`/api/campaigns?${params}`, { signal });
if (!response.ok) throw new Error('Could not refresh campaigns.');
const page = await response.json() as { deployments: T[]; total: number };
total = page.total;
deployments.push(...page.deployments);
if (!page.deployments.length) break;
}
return { deployments, total: Number.isFinite(total) ? total : 0 };
}
+12 -5
View File
@@ -1,25 +1,32 @@
/** Poll visible page content without overlapping requests. Fleet watches use their own monitor. */
export function pollVisible(task: () => void | Promise<unknown>, interval: number, immediate = true): () => void {
export function pollVisible(task: (signal: AbortSignal) => unknown | Promise<unknown>, interval: number, immediate = true): () => void {
let stopped = false;
let running = false;
let timer: ReturnType<typeof setTimeout> | undefined;
let controller: AbortController | undefined;
let failures = 0;
const visible = () => document.visibilityState !== 'hidden';
const run = async () => {
if (stopped || running || !visible()) return;
clearTimeout(timer);
running = true;
try { await task(); }
catch (error) { console.error('Page refresh failed', error); }
controller = new AbortController();
const timeout = setTimeout(() => controller?.abort(), 30_000);
try { failures = await task(controller.signal) === false ? failures + 1 : 0; }
catch (error) { if (!controller.signal.aborted) { failures++; console.error('Page refresh failed', error); } }
finally {
clearTimeout(timeout);
controller = undefined;
running = false;
if (!stopped && visible()) timer = setTimeout(run, interval);
if (!stopped && visible()) timer = setTimeout(run, interval * Math.min(8, 2 ** failures));
}
};
const visibility = () => {
clearTimeout(timer);
if (!visible()) controller?.abort();
if (document.visibilityState !== 'hidden') void run();
};
document.addEventListener('visibilitychange', visibility);
if (immediate) void run(); else timer = setTimeout(run, interval);
return () => { stopped = true; clearTimeout(timer); document.removeEventListener('visibilitychange', visibility); };
return () => { stopped = true; clearTimeout(timer); controller?.abort(); document.removeEventListener('visibilitychange', visibility); };
}
+82 -30
View File
@@ -1,7 +1,9 @@
import { freshStatusTimestamp } from '$lib/server-freshness.js';
import { pollVisible } from '../poll.js';
import { selectServerObservation } from '$lib/server-observation.js';
import { derived, writable } from 'svelte/store';
import { derived, writable, get } from 'svelte/store';
import { browser } from '$app/environment';
import { statusByServer, statusReceivedAt, syncServerTime, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
@@ -11,9 +13,30 @@ export const fleetError = writable('');
export const fleetRefreshing = writable(false);
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, uiNow], ([snapshot, live, received, matches, reports, now]) =>
snapshot.servers.map(server => {
const status = received > snapshot.receivedAt ? live.get(server.name) : undefined;
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
const times = snapshot.servers.flatMap(server => {
const status = live.get(server.name);
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
});
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
});
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
const now = get(uiNow);
const next = snapshot.servers.map(server => {
const candidate = live.get(server.name);
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
const match = matches.get(server.name);
const report = reports.get(server.name);
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
const saved = observed.get(server.name);
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
const base = status ? { ...server, last_players: status.players, last_map: status.map,
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
@@ -23,38 +46,65 @@ export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, ma
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
telemetry: status.telemetry ?? server.telemetry } : server;
const current = selectServerObservation(base, reports.get(server.name), now);
const match = matches.get(server.name);
// Apply lifecycle events immediately, until a newer status observes them
// or confirms that the lobby has emptied.
if (!match || current.health !== 'online'
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|| match.receivedAt < (current.game_started_at ?? '')
|| match.receivedAt < (current.game_ended_at ?? '')) return current;
return { ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
game_ended_at: match.event === 'game_end' ? match.receivedAt : null };
}));
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
});
const names = new Set(snapshot.servers.map(server => server.name));
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
return next;
});
let snapshotServerNow = 0;
export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
if (!browser || snapshot.serverNow < snapshotServerNow) return;
snapshotServerNow = snapshot.serverNow;
syncServerTime(snapshot.serverNow);
fleetIntegrations.set(snapshot.integrations);
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
}
function getIntegrations() { return get(fleetIntegrations); }
let inFlight: Promise<void> | null = null;
export function refreshFleet(): Promise<void> {
let refreshRequested = false;
export function refreshFleet(afterCurrent = false): Promise<void> {
if (!browser) return Promise.resolve();
if (inFlight) return inFlight;
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
inFlight = (async () => {
fleetRefreshing.set(true);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 12_000);
const requestedAt = Date.now();
try {
const responses = await Promise.all([
fetch('/api/servers', { signal: controller.signal }),
fetch('/api/fleet', { signal: controller.signal })
const results = await Promise.allSettled([
(async () => {
const response = await fetch('/api/servers', { signal: controller.signal });
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
const servers = await response.json();
if (servers.serverNow < snapshotServerNow) return;
snapshotServerNow = servers.serverNow;
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
})(),
(async () => {
const response = await fetch('/api/fleet', { signal: controller.signal });
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
const operations = await response.json();
fleet.update(value => ({ ...value, operations: operations.operations }));
})()
]);
if (responses.some(response => !response.ok)) throw new Error('Could not refresh Fleet. Retrying automatically.');
const [servers, operations] = await Promise.all(responses.map(response => response.json()));
syncServerTime(servers.serverNow);
const failure = results.find(result => result.status === 'rejected');
if (failure?.status === 'rejected') throw failure.reason;
fleetError.set('');
fleetIntegrations.set(servers.integrations);
fleet.set({ servers: servers.servers, operations: operations.operations, receivedAt: requestedAt });
} catch (error) {
fleetError.set(error instanceof Error && error.name !== 'AbortError'
? error.message : 'Fleet refresh timed out. Retrying automatically.');
@@ -62,23 +112,25 @@ export function refreshFleet(): Promise<void> {
clearTimeout(timeout);
fleetRefreshing.set(false);
}
})().finally(() => { inFlight = null; });
})().finally(() => {
inFlight = null;
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
});
return inFlight;
}
let stop: (() => void) | null = null;
export function startFleetMonitor(): () => void {
if (!browser || stop) return () => {};
void refreshFleet();
const interval = setInterval(() => {
if (document.visibilityState !== 'hidden') void refreshFleet();
}, 10_000);
const onFocus = () => { if (document.visibilityState === 'visible') void refreshFleet(); };
document.addEventListener('visibilitychange', onFocus);
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
const changed = () => {
if (refreshTimer || document.visibilityState === 'hidden') return;
refreshTimer = setTimeout(() => { refreshTimer = undefined; void refreshFleet(true); }, 250);
};
window.addEventListener('ec:fleet-changed', changed);
stop = () => {
clearInterval(interval);
document.removeEventListener('visibilitychange', onFocus);
stop = null;
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
};
return stop;
}
+3
View File
@@ -26,6 +26,9 @@ function updateDeployment(id: string, mutate: (d: PipelineDeployment) => void):
export function startPipelineStream(): (() => void) | undefined {
if (!browser || source) return;
source = new EventSource('/api/events/pipeline');
for (const event of ['server-changed', 'server-state', 'deployment-opened', 'fleet-invalidated']) {
source.addEventListener(event, () => window.dispatchEvent(new Event('ec:fleet-changed')));
}
source.addEventListener('deployment-opened', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { deploymentId: string; action: string; servers: string[] };
+6 -5
View File
@@ -17,6 +17,7 @@ export interface RconEndpoint {
}
export interface ServerStatus {
receivedAt?: number;
server: string;
containerState?: string | null;
containerCheckedAt?: string | null;
@@ -91,17 +92,17 @@ export function startServerStream(): (() => void) | undefined {
});
});
source.addEventListener('server-status', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number };
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number; delta?: boolean };
syncServerTime(data.serverNow);
statusByServer.update(() => {
const m = new Map<string, ServerStatus>();
for (const e of data.entries) m.set(e.server, e);
statusByServer.update(previous => {
const m = data.delta ? new Map(previous) : new Map<string, ServerStatus>();
for (const e of data.entries) m.set(e.server, { ...e, receivedAt: Date.now() });
return m;
});
statusReceivedAt.set(Date.now());
});
if (tickInterval === null) {
tickInterval = setInterval(() => _uiNow.set(serverTimeNow()), 1000);
tickInterval = setInterval(() => { if (typeof document === 'undefined' || document.visibilityState !== 'hidden') _uiNow.set(serverTimeNow()); }, 1000);
}
return () => { source?.close(); source = null; if (tickInterval) clearInterval(tickInterval); tickInterval = null; };
}
+13 -4
View File
@@ -1,10 +1,10 @@
<script lang="ts">
import { onMount } from 'svelte';
import { startTimeZonePreference } from '$ui/stores/time-zone.js';
import { afterNavigate, onNavigate, goto } from '$app/navigation';
import { afterNavigate, beforeNavigate, onNavigate, goto } from '$app/navigation';
import { startPipelineStream } from '$ui/stores/pipeline.js';
import { startServerStream } from '$ui/stores/servers.js';
import { startFleetMonitor } from '$ui/stores/fleet.js';
import { startFleetMonitor, hydrateFleet } from '$ui/stores/fleet.js';
import { page } from '$app/stores';
import BrandMark from '$ui/components/BrandMark.svelte';
import Toaster from '$ui/components/Toaster.svelte';
@@ -40,6 +40,8 @@
else helpOpen = true;
}
beforeNavigate(() => performance.mark('ec:navigation-start'));
afterNavigate(() => { if (performance.getEntriesByName('ec:navigation-start').length) { performance.measure('ec:navigation', 'ec:navigation-start'); performance.clearMarks('ec:navigation-start'); } });
afterNavigate(() => { menuOpen = false; accountOpen = false; resolveConfirmation(false); });
function closeMenu(event: KeyboardEvent) {
@@ -67,8 +69,15 @@
const stopNotifications = startNotificationHistory();
const stopPipeline = startPipelineStream();
const stopServers = startServerStream();
const stopFleet = startFleetMonitor();
return () => { stopNotifications(); stopFleet(); stopPipeline?.(); stopServers?.(); };
return () => { stopNotifications(); stopPipeline?.(); stopServers?.(); };
});
$effect(() => {
if (!sessionReady) return;
const path = $page.url.pathname;
if (!(path === '/' || /^\/(servers|scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
if ('fleetSnapshot' in $page.data && $page.data.fleetSnapshot) hydrateFleet($page.data.fleetSnapshot);
return startFleetMonitor();
});
onMount(() => {
+4
View File
@@ -0,0 +1,4 @@
import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
import { fleetOperations } from '$lib/server/fleet/overview.js';
export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }
+21 -17
View File
@@ -1,4 +1,7 @@
<script lang="ts">
let { data }: { data: import('./$types').PageData } = $props();
const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
import { onMount, untrack } from 'svelte';
import { afterNavigate, replaceState } from '$app/navigation';
import { page } from '$app/stores';
@@ -60,7 +63,7 @@
} catch { /* Preferences also work when storage is disabled. */ }
readViewUrl();
preferencesReady = true;
void refreshFleet();
});
$effect(() => {
@@ -75,25 +78,26 @@
if (url.href !== window.location.href) replaceState(url, $page.state);
});
});
const snapshotAge = $derived.by(() => { $uiNow; return Math.max(0, Math.floor((Date.now() - $fleet.receivedAt) / 1000)); });
const snapshotAge = $derived.by(() => { $uiNow; return Math.max(0, Math.floor((Date.now() - ($fleet.receivedAt || data.fleetSnapshot.serverNow)) / 1000)); });
const integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
let servers = $derived($fleetServers.map(server => server.health === 'online' && !freshServer(server, $uiNow)
? { ...server, health: 'stale' as const } : server));
let activeOperations = $derived($fleet.operations.filter(op => op.state !== 'done'));
let servers = $derived(visibleFleet);
const observationNow = () => untrack(() => $fleet.receivedAt ? $uiNow : data.fleetSnapshot.serverNow);
let activeOperations = $derived(snapshot.operations.filter(op => op.state !== 'done'));
let attentionOperations = $derived(activeOperations.filter(op => op.state === 'attention'));
let campaigns = $derived(new Set(activeOperations.filter(op => op.kind === 'campaign').map(op => op.group)).size);
let healthy = $derived(servers.filter(server => freshServer(server, $uiNow)));
let healthy = $derived(servers.filter(server => freshServer(server, observationNow())));
let players = $derived(healthy.reduce((sum, server) => sum + (server.last_players ?? 0), 0));
let empty = $derived(healthy.filter(server => server.last_players === 0).length);
let problems = $derived(servers.filter(server => server.health === 'offline' || server.health === 'stale'));
let starting = $derived(servers.filter(server => server.health === 'starting').length);
let provisioning = $derived(servers.filter(server => server.health === 'provisioning').length);
let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, $uiNow))
let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, observationNow()))
.sort((a, b) => {
const pinOrder = Number(pinned.includes(b.name)) - Number(pinned.includes(a.name));
if (pinOrder) return pinOrder;
const rank = (server: FleetServer) => sort === 'players' ? (freshServer(server, $uiNow) ? server.last_players ?? -1 : -1)
: sort === 'round' ? currentRound(server, $telemetryByServer.get(server.name), $uiNow) ?? -1
const rank = (server: FleetServer) => sort === 'players' ? (freshServer(server, observationNow()) ? server.last_players ?? -1 : -1)
: sort === 'round' ? currentRound(server, $telemetryByServer.get(server.name), observationNow()) ?? -1
: sort === 'duration' ? matchDuration(server) : 0;
return rank(b) - rank(a) || serverTitle(a.name).localeCompare(serverTitle(b.name), undefined, { numeric: true });
}));
@@ -109,12 +113,12 @@
}
return [...result.entries()].sort(([, a], [, b]) => a.title.localeCompare(b.title));
});
let spotlight = $derived(servers.filter(server => currentRound(server, $telemetryByServer.get(server.name), $uiNow) !== null)
let spotlight = $derived(servers.filter(server => currentRound(server, $telemetryByServer.get(server.name), observationNow()) !== null)
.sort((a, b) => matchDuration(b) - matchDuration(a))[0]);
function matchDuration(server: FleetServer): number {
return freshServer(server, $uiNow) && server.active_since && server.game_started_at && !server.game_ended_at
&& (server.last_players ?? 0) > 0 ? Math.max(0, $uiNow - Date.parse(server.game_started_at)) : -1;
return freshServer(server, observationNow()) && server.active_since && server.game_started_at && !server.game_ended_at
&& (server.last_players ?? 0) > 0 ? Math.max(0, observationNow() - Date.parse(server.game_started_at)) : -1;
}
function togglePin(name: string) { pinned = pinned.includes(name) ? pinned.filter(item => item !== name) : [...pinned, name]; }
function launch(selection: { game: ZombieGame; slug: string } | null = null) {
@@ -139,7 +143,7 @@
<button onclick={() => void refreshFleet()}>Retry now</button></div>
{/if}
{#if !$fleet.receivedAt}
{#if !$fleet.receivedAt && !data.fleetSnapshot}
{#if !$fleetError}<div class="loading" role="status">Loading your fleet…</div>{/if}
{:else}
{#if servers.length}<div class="overview" aria-label="Fleet summary">
@@ -149,10 +153,10 @@
<a class="metric-link" class:no-campaigns={!campaigns} href="/campaigns"><strong>{campaigns}</strong> active campaigns</a>
<span class="freshness">{$fleetError ? 'Reconnecting' : 'Updated'} · {snapshotAge < 60 ? `${snapshotAge}s` : `${Math.floor(snapshotAge / 60)}m`} ago</span>
</div>{/if}
{#if $fleetIntegrations?.native && !$fleetIntegrations.hostsReady}
<div class="setup-note"><span><strong>Prepare a game host</strong> · {$fleetIntegrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}</span><a href="/hosts">Review hosts →</a></div>
{:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !$fleetIntegrations.native && (!$fleetIntegrations.komodo || !$fleetIntegrations.rcon)}
<div class="setup-note"><span><strong>Finish system setup</strong> · {!$fleetIntegrations.komodo ? 'Komodo' : ''}{!$fleetIntegrations.komodo && !$fleetIntegrations.rcon ? ' and ' : ''}{!$fleetIntegrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
{#if integrations?.native && !integrations.hostsReady}
<div class="setup-note"><span><strong>Prepare a game host</strong> · {integrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}</span><a href="/hosts">Review hosts →</a></div>
{:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !integrations.native && (!integrations.komodo || !integrations.rcon)}
<div class="setup-note"><span><strong>Finish system setup</strong> · {!integrations.komodo ? 'Komodo' : ''}{!integrations.komodo && !integrations.rcon ? ' and ' : ''}{!integrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
{/if}
{#if problems.length || attentionOperations.length || starting || provisioning}
+11 -3
View File
@@ -1,9 +1,17 @@
import { json } from '@sveltejs/kit';
import { pendingHostOperations } from '$lib/server/hosts/index.js';
import { pendingHostOperations, waitForHostWork } from '$lib/server/hosts/index.js';
import { authenticateAgent } from '$lib/server/hosts/http.js';
import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
export function GET({ request }: { request: Request }) {
export async function GET({ request }: { request: Request }) {
const host = authenticateAgent(request);
reconcileHostPreparation(host.id);
return json({ operations: pendingHostOperations(host.id), serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } });
const params = new URL(request.url).searchParams;
const lane = params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
let operations = pendingHostOperations(host.id, lane);
if (!operations.length && params.get('wait') === '20') {
await waitForHostWork(host.id, request.signal);
authenticateAgent(request); // Revocation still applies after waiting.
operations = pendingHostOperations(host.id, lane);
}
return json({ operations, serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } });
}
+19 -6
View File
@@ -1,9 +1,22 @@
import { json } from '@sveltejs/kit';
import { serverTitle } from '$lib/ui/server-labels.js';
import { db } from '$lib/server/db/index.js';
export async function GET({ url }: { url: URL }) {
const activeOnly = url.searchParams.get('active') === 'true';
const historyOnly = url.searchParams.get('view') === 'history';
const server = url.searchParams.get('server');
const query = (url.searchParams.get('q') ?? '').slice(0, 100).toLowerCase();
const limit = Math.min(500, Math.max(1, Math.floor(Number(url.searchParams.get('limit')) || 25)));
const offset = Math.min(1_000_000, Math.max(0, Math.floor(Number(url.searchParams.get('offset')) || 0)));
const active = `EXISTS (SELECT 1 FROM deployment_servers a WHERE a.deployment_id=d.id AND a.server_deleted_at IS NULL AND a.restart_state IN ('pending','failed'))`;
const names = query ? (db().prepare('SELECT DISTINCT server FROM deployment_servers').all() as { server: string }[])
.map(row => row.server).filter(name => `${name} ${serverTitle(name)}`.toLowerCase().includes(query)) : [];
const where = `(? IS NULL OR EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id=d.id AND target.server=?))
${activeOnly ? `AND ${active}` : historyOnly ? `AND NOT ${active}` : ''}
${query ? `AND EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id=d.id AND target.server IN (${names.map(() => '?').join(',') || 'NULL'}))` : ''}`;
const parameters = [server, server, ...names];
const { total } = db().prepare(`SELECT COUNT(*) AS total FROM deployments d WHERE ${where}`).get(...parameters) as { total: number };
const deployments = db()
.prepare(
@@ -15,11 +28,11 @@ export async function GET({ url }: { url: URL }) {
LEFT JOIN scripts s ON s.id = d.script_id
LEFT JOIN script_versions sv ON sv.id = d.script_version_id
LEFT JOIN script_versions pv ON pv.id = d.prior_version_id
WHERE (? IS NULL OR EXISTS (SELECT 1 FROM deployment_servers target WHERE target.deployment_id = d.id AND target.server = ?))
ORDER BY d.started_at DESC
LIMIT 500`
WHERE ${where}
ORDER BY d.started_at DESC, d.rowid DESC
LIMIT ? OFFSET ?`
)
.all(server, server) as Array<{
.all(...parameters, limit, offset) as Array<{
id: string;
action: string;
started_at: string;
@@ -36,7 +49,7 @@ export async function GET({ url }: { url: URL }) {
}>;
const ids = deployments.map(deployment => deployment.id);
if (!ids.length) return json({ deployments: [] });
if (!ids.length) return json({ deployments: [], total, limit, offset });
const placeholders = ids.map(() => '?').join(',');
const serversByDep = db()
.prepare(
@@ -95,5 +108,5 @@ export async function GET({ url }: { url: URL }) {
}))
.filter((d) => (activeOnly ? d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed')) : true));
return json({ deployments: rows });
return json({ deployments: rows, total, limit, offset });
}
+1 -1
View File
@@ -21,7 +21,7 @@ export async function GET() {
event.type === 'server-state' ||
event.type === 'deployment-opened' ||
event.type === 'server-changed' ||
event.type === 'sweep-tick'
event.type === 'sweep-tick' || event.type === 'fleet-invalidated'
) {
send(controller, event);
}
+6
View File
@@ -1,3 +1,6 @@
import { observationReader } from '$lib/server/fleet/status.js';
import { listServers } from '$lib/server/fleet/servers.js';
import { statusEntry } from '$lib/server/pipeline/worker.js';
import { subscribe } from '$lib/server/pipeline/bus.js';
import { listTelemetry } from '$lib/server/fleet/telemetry.js';
@@ -15,7 +18,10 @@ export async function GET() {
controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, serverNow: Date.now() })}\n\n`));
}
});
const observation = observationReader();
const observe = (server: ReturnType<typeof listServers>[number]) => statusEntry(server, observation);
controller.enqueue(encoder.encode(': hello\n\n'));
controller.enqueue(encoder.encode(`event: server-status\ndata: ${JSON.stringify({ entries: listServers().map(server => observe(server)), snapshot: true, serverNow: Date.now() })}\n\n`));
controller.enqueue(encoder.encode(`event: server-telemetry\ndata: ${JSON.stringify({ entries: listTelemetry(), snapshot: true, serverNow: Date.now() })}\n\n`));
heartbeat = setInterval(() => controller.enqueue(encoder.encode(': ping\n\n')), HEARTBEAT_MS);
},
+1 -1
View File
@@ -8,7 +8,7 @@ export function GET({ request, params }: Event) {
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
// Payloads and results can contain RCON keys or staged credentials. History
// exposes metadata only; agent APIs remain the only command delivery path.
return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) });
}
export async function PATCH({ request, params }: Event) {
requireManagementOrigin(request);
+4 -3
View File
@@ -1,7 +1,8 @@
import { leaderboardReport } from '$lib/server/reports/index.js';
import { json } from '@sveltejs/kit';
import { getLeaderboard, isLeaderboardMetric, MAX_LEADERBOARD_OFFSET } from '$lib/server/fleet/leaderboard.js';
import { isLeaderboardMetric, MAX_LEADERBOARD_OFFSET } from '$lib/server/fleet/leaderboard.js';
export function GET({ url }: { url: URL }) {
export async function GET({ url }: { url: URL }) {
const params = url.searchParams;
const game = params.get('game');
const map = params.get('map');
@@ -20,7 +21,7 @@ export function GET({ url }: { url: URL }) {
|| !Number.isFinite(offset) || offset < 0 || offset > MAX_LEADERBOARD_OFFSET) {
return json({ ok: false, kind: 'bad-request', error: 'Use game t5 or t6, a supported metric, map key up to 1024 characters, q up to 100 characters, limit 1–100, and offset 0–1000000.' }, { status: 400 });
}
return json(getLeaderboard({ game: game ?? undefined, map: map || undefined,
return json(await leaderboardReport({ game: game ?? undefined, map: map || undefined,
metric: metric !== null && isLeaderboardMetric(metric) ? metric : undefined,
q: q ?? undefined, limit, offset }));
}
+3 -3
View File
@@ -1,9 +1,9 @@
import { json } from '@sveltejs/kit';
import { parseMetricsQuery } from '$lib/metrics.js';
import { readMetrics } from '$lib/server/fleet/metrics.js';
import { metricsReport } from '$lib/server/reports/index.js';
export function GET({ url }: { url: URL }) {
return json(readMetrics(parseMetricsQuery(url.searchParams)), {
export async function GET({ url }: { url: URL }) {
return json(await metricsReport(parseMetricsQuery(url.searchParams)), {
headers: { 'cache-control': 'no-store' }
});
}
+2 -40
View File
@@ -1,48 +1,10 @@
import { db } from '$lib/server/db/index.js';
import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
import { kick } from '$lib/server/pipeline/worker.js';
import { json } from '@sveltejs/kit';
import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
import { serverObservation } from '$lib/server/fleet/status.js';
import { queueCreateServer } from '$lib/server/provisioning/index.js';
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
import type { GameMode, ZombieGame } from '$lib/server/fleet/map-aliases.js';
import { runningVersions } from '$lib/server/integrations/plutonium.js';
import { listHosts } from '$lib/server/hosts/index.js';
import { nativeInstance } from '$lib/server/backends/native.js';
export async function GET() {
const rconHost = config.rcon.host;
const hosts = listHosts();
const hostsById = new Map(hosts.map(host => [host.id, host]));
const versions = new Map(runningVersions().map(version => [version.server, version]));
const servers = listServers().map((s) => {
const instance = s.native_instance_id ? nativeInstance(s.name) : null;
return {
...serverObservation(s),
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
};
});
return json({
servers,
serverNow: Date.now(),
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
komodo: komodoConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
defaultRconHost: rconHost
}
});
}
export function GET() { return json(fleetSnapshot()); }
export async function POST({ request }: { request: Request }) {
const body = (await request.json().catch(() => ({}))) as {
+1 -1
View File
@@ -176,7 +176,7 @@
<div class="page">
<SectionHeader title="New beta" subtitle="Choose exact file versions and test servers, then review before starting.">{#snippet actions()}<Button href="/betas" variant="ghost">Back to betas</Button>{/snippet}</SectionHeader>
{#if flash}<div class="ui-error" role="alert">{flash}{#if !scripts.length}<Button size="sm" onclick={()=>initialize()}>Retry</Button>{/if}</div>{/if}
{#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={refreshFleet}>Retry servers</Button></div>{/if}
{#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={() => refreshFleet(true)}>Retry servers</Button></div>{/if}
{#if notice}<p role="status" class="notice">{notice}</p>{/if}
{#if loading}<EmptyState title="Loading beta setup…"/>
{:else if availableDraft}<section class="ui-panel draft"><h2>Resume unfinished beta?</h2><p>Saved {$timeFormat.dateTime(availableDraft.savedAt)} · {availableDraft.picks.filter(p=>p.scriptId!==null).length} files · {availableDraft.servers.length} selected servers. Versions and eligibility will be rechecked.</p><div class="actions"><Button variant="primary" onclick={resumeDraft}>Resume draft</Button><Button onclick={discardDraft}>{hasLinkedSetup ? 'Discard draft and use linked files' : 'Discard draft'}</Button></div></section>
+22 -9
View File
@@ -4,6 +4,7 @@
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { pollVisible } from '$ui/poll.js';
import { fetchCampaigns } from '$ui/campaigns.js';
import { confirmAction } from '$ui/confirm.js';
import { serverTitle } from '$ui/server-labels.js';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
@@ -48,6 +49,8 @@
}
let loadError = $state('');
let limit = $state(25);
let historyTotal = $state(0);
let loadSequence = 0;
let deployments = $state<Deployment[]>([]);
let loading = $state(true);
let rollingBack = $state<string | null>(null);
@@ -67,15 +70,25 @@
void goto(url, { replaceState: true, noScroll: true, keepFocus: true });
}
async function loadCampaigns(): Promise<void> {
async function loadCampaigns(signal?: AbortSignal): Promise<boolean> {
const sequence = ++loadSequence;
try {
const res = await fetch('/api/campaigns');
if (!res.ok) throw new Error('Could not refresh campaigns.');
deployments = (await res.json()).deployments; loadError = '';
} catch (error) { loadError = (error as Error).message; }
finally { loading = false; }
const [history, active] = await Promise.all([
fetchCampaigns<Deployment>({ q: serverFilter, view: 'history' }, limit, signal),
fetchCampaigns<Deployment>({ q: serverFilter, active: 'true' }, Infinity, signal)
]);
if (sequence !== loadSequence || signal?.aborted) return true;
deployments = [...active.deployments, ...history.deployments]; historyTotal = history.total; loadError = '';
return true;
} catch (error) {
if (sequence === loadSequence && !signal?.aborted) loadError = (error as Error).message;
return false;
} finally { if (sequence === loadSequence && !signal?.aborted) loading = false; }
}
onMount(() => pollVisible(loadCampaigns, 5000));
$effect(() => {
void limit; void serverFilter;
return pollVisible(loadCampaigns, 15000);
});
function isActive(d: Deployment): boolean {
return d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'));
@@ -145,7 +158,7 @@
subtitle="Track file campaigns, server activation, and rollbacks."
/>
{#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={loadCampaigns}>Retry</Button></div>{/if}
{#if loadError}<div class="ui-error" role="alert">{loadError}<Button size="sm" onclick={() => loadCampaigns()}>Retry</Button></div>{/if}
{#if loading}
<div class="loading">Loading…</div>
{:else}
@@ -195,7 +208,7 @@
{/each}
{/if}
</section>
<ListFooter total={history.length} bind:limit onchange={() => saveContext()} label="campaigns" />
<ListFooter total={historyTotal} bind:limit onchange={() => saveContext()} label="campaigns" />
{/if}
</div>
+35 -24
View File
@@ -26,7 +26,9 @@
let phase = $state<Phase>('idle'); let currentPath = $state(''); let pauseRequested = $state(false);
let acceptedBytes = $state(0); let sendingBytes = $state(0); let completedFiles = $state(0);
let folderInput = $state<HTMLInputElement>();
let activeRequest: XMLHttpRequest | null = null; let disposed = false;
const activeRequests = new Map<XMLHttpRequest, number>(); let disposed = false; let failedTransfer = false;
const updateSending = () => { sendingBytes = [...activeRequests.values()].reduce((a, b) => a + b, 0); };
const abortUploads = () => { for (const request of activeRequests.keys()) request.abort(); };
const canManage = $derived($page.data.user?.role === 'admin');
const onlineHosts = $derived(hosts.filter(host => host.connected && host.credentialActive && host.mode !== 'disabled'));
const totalBytes = $derived(folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0);
@@ -122,21 +124,21 @@
finally { uploadBusy = false; }
await tick(); document.getElementById('game-upload')?.scrollIntoView({ behavior: 'smooth' });
}
function pauseUpload() { pauseRequested = true; activeRequest?.abort(); }
function pauseUpload() { pauseRequested = true; abortUploads(); }
async function sendChunk(path: string, file: File, offset: number, length: number): Promise<number> {
const chunk = file.slice(offset, offset + length);
const hash = await crypto.subtle.digest('SHA-256', await chunk.arrayBuffer());
if (pauseRequested || disposed) throw new Error('Upload paused.');
if (pauseRequested || disposed || failedTransfer) throw new Error('Upload paused.');
const sha256 = [...new Uint8Array(hash)].map(value => value.toString(16).padStart(2, '0')).join('');
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest(); activeRequest = request;
const request = new XMLHttpRequest(); activeRequests.set(request, 0);
request.open('PUT', `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}`);
request.setRequestHeader('Content-Type', 'application/octet-stream');
request.setRequestHeader('X-Upload-Offset', String(offset)); request.setRequestHeader('X-Upload-Length', String(length));
request.setRequestHeader('X-Content-SHA256', sha256); request.timeout = 300_000;
request.upload.onprogress = event => { sendingBytes = Math.min(length, event.loaded); };
request.upload.onprogress = event => { activeRequests.set(request, Math.min(length, event.loaded)); updateSending(); };
request.onload = () => {
activeRequest = null;
activeRequests.delete(request); updateSending();
try {
const body = JSON.parse(request.responseText);
if (request.status < 200 || request.status >= 300) throw new Error(body.error ?? `Upload failed (${request.status}).`);
@@ -144,16 +146,16 @@
resolve(body.receivedBytes);
} catch (cause) { reject(cause); }
};
request.onerror = () => { activeRequest = null; reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
request.ontimeout = () => { activeRequest = null; reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
request.onabort = () => { activeRequest = null; reject(new Error('Upload paused.')); };
request.onerror = () => { activeRequests.delete(request); updateSending(); reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
request.ontimeout = () => { activeRequests.delete(request); updateSending(); reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
request.onabort = () => { activeRequests.delete(request); updateSending(); reject(new Error('Upload paused.')); };
request.send(chunk);
});
}
async function startUpload(event: SubmitEvent) {
event.preventDefault(); if (uploadBusy || !folder) return;
if (!globalThis.crypto?.subtle) { uploadError = 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; return; }
uploadBusy = true; pauseRequested = false; uploadError = ''; notice = ''; phase = 'preparing';
uploadBusy = true; pauseRequested = false; failedTransfer = false; uploadError = ''; notice = ''; phase = 'preparing';
try {
if (!uploadId) {
const result = await responseBody(await fetch('/api/game-files', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, files: folder.files.map(({ path, file }) => ({ path, size: file.size, lastModified: file.lastModified })) }) }));
@@ -165,19 +167,28 @@
const localFiles = new Map(folder.files.map(item => [item.path, item.file]));
const chunkSize = Math.min(8_388_608, Math.max(1, status.chunkSize));
phase = 'uploading';
for (const remote of status.files) {
if (pauseRequested || disposed) break;
if (remote.complete) continue;
currentPath = remote.path;
const file = localFiles.get(remote.path)!;
do {
if (pauseRequested || disposed) break;
const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
acceptedBytes += length; sendingBytes = 0;
if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
} while (!remote.complete);
}
let nextFile = 0;
let failure: unknown;
const transfer = async () => {
while (!pauseRequested && !disposed && !failure) {
const remote = status.files[nextFile++];
if (!remote) return;
if (remote.complete) continue;
currentPath = remote.path;
const file = localFiles.get(remote.path)!;
try {
do {
if (pauseRequested || disposed || failure) return;
const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
acceptedBytes += length;
if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
} while (!remote.complete);
} catch (cause) { failure ??= cause; failedTransfer = true; abortUploads(); }
}
};
await Promise.all(Array.from({ length: Math.min(3, status.files.length) }, transfer));
if (failure) throw failure;
if (pauseRequested || disposed) { phase = 'paused'; return; }
phase = 'verifying'; currentPath = '';
const verification = await responseBody(await fetch(`/api/game-files/${uploadId}/upload/complete`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })) as UploadStatus;
@@ -192,7 +203,7 @@
});
onMount(() => {
const stop = pollVisible(load, 5000);
return () => { disposed = true; pauseRequested = true; activeRequest?.abort(); stop(); };
return () => { disposed = true; pauseRequested = true; abortUploads(); stop(); };
});
</script>
+4
View File
@@ -0,0 +1,4 @@
import { listHosts } from '$lib/server/hosts/index.js';
import { presentHostPreparation } from '$lib/server/hosts/preparation.js';
export function load() { return { hosts: listHosts().map(presentHostPreparation) }; }
+9 -7
View File
@@ -1,22 +1,24 @@
<script lang="ts">
import { onMount } from 'svelte';
import { onMount, untrack } from 'svelte';
import { pollVisible } from '$ui/poll.js';
let { data }: { data: import('./$types').PageData } = $props();
import { page } from '$app/stores';
import type { Host } from '$lib/hosts.js';
import { hostAgentUpdateLabel, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
let hosts = $state<Host[]>([]);
let loading = $state(true); let busy = $state(false); let error = $state('');
let hosts = $state<Host[]>(untrack(() => data.hosts));
let loading = $state(false); let busy = $state(false); let error = $state('');
let notice = $state(''); let deleteOpen = $state(false); let deletingHost = $state<Host | null>(null);
let loadSequence = 0;
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980);
let enrollment = $state<{ command: string; expiresAt: string; hostId: string } | null>(null);
const canManage = $derived($page.data.user?.role === 'admin');
async function load() {
async function load(signal?: AbortSignal) {
const sequence = ++loadSequence;
try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
try { const response = await fetch('/api/hosts', { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
catch (cause) { if (!signal?.aborted && sequence === loadSequence) error = (cause as Error).message; return false; } finally { if (sequence === loadSequence) loading = false; }
}
async function deleted(result: { deleted: boolean }) {
notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
@@ -32,7 +34,7 @@
enrollment = body.enrollment; name = ''; await load();
} catch (cause) { error = (cause as Error).message; } finally { busy = false; }
}
onMount(() => { void load(); const timer = setInterval(() => { void load(); }, 15000); return () => clearInterval(timer); });
onMount(() => pollVisible(load, 30000, false));
</script>
<svelte:head><title>Hosts · Electric Cherry</title></svelte:head>
+14 -5
View File
@@ -1,5 +1,7 @@
<script lang="ts">
import { onMount } from 'svelte';
import { untrack } from 'svelte';
import { pollVisible } from '$ui/poll.js';
let requestSequence = 0;
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
@@ -17,11 +19,12 @@
const canManage = $derived($page.data.user?.role === 'admin');
const cleaning = $derived(!!host?.decommission);
function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
async function load(initialize = false) {
async function load(initialize = false, signal?: AbortSignal) {
const target = endpoint; const sequence = ++requestSequence;
try {
const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; error = '';
if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; }
} catch (cause) { error = (cause as Error).message; } finally { loading = false; }
} catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; }
}
async function action(path: string, method: string, body?: unknown, message?: string) {
busy = true; error = ''; notice = '';
@@ -30,7 +33,13 @@
}
async function save(event: SubmitEvent) { event.preventDefault(); await action('', 'PATCH', { name, portMin, portMax, bindAddress, publicAddress, minimumFreeBytes: Math.round(minimumFreeGB * 1073741824) }, 'Host settings saved.'); }
async function mode(value: HostMode) { await action('', 'PATCH', { mode: value }, value === 'draining' ? 'Drain enabled. Existing games continue until safe retirement.' : `Host ${value === 'active' ? 'enabled' : 'disabled'}.`); }
onMount(() => { void load(true); const timer = setInterval(() => { void load(); }, 10000); return () => clearInterval(timer); });
$effect(() => {
void endpoint;
return untrack(() => {
let initialize = true;
return pollVisible(async signal => { const result = await load(initialize, signal); if (result !== false) initialize = false; return result; }, 15000);
});
});
</script>
<svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>
+10
View File
@@ -0,0 +1,10 @@
import { GET } from '../api/leaderboard/+server.js';
import { error } from '@sveltejs/kit';
import type { LeaderboardResponse } from '$lib/leaderboard.js';
export async function load({ url }: { url: URL }) {
const response = await GET({ url });
const body = await response.json();
if (!response.ok) error(response.status, body.error);
return { leaderboard: body as LeaderboardResponse };
}
+12 -10
View File
@@ -1,8 +1,10 @@
<script lang="ts">
let { data: initial }: { data: import('./$types').PageData } = $props();
import { timeFormat } from '$ui/stores/time-zone.js';
import { onMount, untrack } from 'svelte';
import { goto } from '$app/navigation';
import { page } from '$app/stores';
import { page, navigating } from '$app/stores';
const changingFilters = $derived($navigating?.to?.route.id === '/leaderboard');
import { leaderboardMetrics, type LeaderboardMetric, type PlayerStatMetric, type LeaderboardEntry, type LeaderboardMap, type LeaderboardResponse } from '$lib/leaderboard.js';
import { playerContextHref } from '$lib/leaderboard-navigation.js';
import { efficiencyMetrics, isEfficiencyMetric, efficiencyRequirement, efficiencySample, formatEfficiencyValue } from '$lib/leaderboard-efficiency.js';
@@ -33,9 +35,9 @@
const formatValue = (entry: LeaderboardEntry, key: LeaderboardMetric) => isEfficiencyMetric(key)
? formatEfficiencyValue(entry[key], key) : number.format(entry[key]);
let data = $state<LeaderboardResponse | null>(null);
let maps = $state<LeaderboardMap[]>([]);
let loading = $state(true);
let data = $state<LeaderboardResponse | null>(untrack(() => initial.leaderboard));
let maps = $state<LeaderboardMap[]>(untrack(() => initial.leaderboard.maps));
let loading = $state(false);
let error = $state('');
let search = $state('');
let mounted = $state(false);
@@ -68,8 +70,8 @@
$effect(() => { search = query; });
$effect(() => {
const url = apiUrl;
if (mounted) untrack(() => { void refresh(url); });
const report = initial.leaderboard;
untrack(() => { ++requestId; controller?.abort(); data = report; maps = report.maps; loadedUrl = apiUrl; loading = false; error = ''; });
});
onMount(() => {
@@ -91,7 +93,7 @@
controller?.abort();
const activeController = new AbortController();
controller = activeController;
if (url !== loadedUrl) { data = null; checkedAt = null; }
if (url !== loadedUrl) checkedAt = null;
loading = true;
error = '';
const timeout = setTimeout(() => activeController.abort(), 20_000);
@@ -191,10 +193,10 @@
</div>
</details>
<section class="rankings" class:compact={!fullTable} aria-labelledby="rankings-title" aria-busy={loading}>
<section class="rankings" class:compact={!fullTable} aria-labelledby="rankings-title" aria-busy={loading || changingFilters}>
<div class="rankings-header">
<div><h2 id="rankings-title">{metricLabel}</h2><p>{description(metric)}</p>{#if isEfficiencyMetric(metric)}<p>Qualification is measured within the selected game and map. Fleet totals include all recorded players.</p>{/if}</div>
<div class="refresh-status" role="status">{loading ? 'Refreshing…' : checkedAt ? `Checked ${$timeFormat.time(checkedAt)}` : ''}<span>Refreshes every 15 seconds</span></div>
<div class="refresh-status" role="status">{changingFilters ? 'Updating filters…' : loading ? 'Refreshing…' : checkedAt ? `Checked ${$timeFormat.time(checkedAt)}` : ''}<span>Refreshes every 15 seconds</span></div>
</div>
{#if data.entries.length}
<div class="mobile-view"><button class="text-button" aria-pressed={fullTable} onclick={() => fullTable = !fullTable}>{fullTable ? 'Show selected stat only' : 'Show all stats and player IDs'}</button></div>
@@ -227,7 +229,7 @@
</div>
<div class="pagination">
<span>{number.format(data.offset + 1)}–{number.format(data.offset + data.entries.length)} of {number.format(data.total)} {isEfficiencyMetric(metric) ? 'qualifying ' : ''}{data.total === 1 ? 'player' : 'players'}{query ? ` matching “${query}”` : ''}</span>
<div><Button size="sm" disabled={data.offset === 0 || loading} onclick={() => updateFilters({ offset: String(Math.max(0, offset - pageSize)) })}>Previous</Button><Button size="sm" disabled={data.offset + data.entries.length >= data.total || loading} onclick={() => updateFilters({ offset: String(offset + pageSize) })}>Next</Button></div>
<div><Button size="sm" disabled={data.offset === 0 || loading || changingFilters} onclick={() => updateFilters({ offset: String(Math.max(0, offset - pageSize)) })}>Previous</Button><Button size="sm" disabled={data.offset + data.entries.length >= data.total || loading || changingFilters} onclick={() => updateFilters({ offset: String(offset + pageSize) })}>Next</Button></div>
</div>
{:else if isEfficiencyMetric(metric)}
<EmptyState spacious title="No qualifying players found" description={`${efficiencyRequirement(metric)} in this game/map selection. Try another scope or player search.`}>
@@ -1,9 +1,9 @@
import { error } from '@sveltejs/kit';
import { getPlayerProfile } from '$lib/server/fleet/player-profile.js';
import { getPlayerLeaderboardEntry, isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js';
import { playerReport } from '$lib/server/reports/index.js';
import { isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js';
import { leaderboardReturnHref } from '$lib/leaderboard-navigation.js';
export function load({ params, url }: { params: { guid: string }; url: URL }) {
export async function load({ params, url }: { params: { guid: string }; url: URL }) {
const game = url.searchParams.get('game');
const map = url.searchParams.get('map');
const requestedMetric = url.searchParams.get('metric');
@@ -13,8 +13,8 @@ export function load({ params, url }: { params: { guid: string }; url: URL }) {
}
const metric = requestedMetric !== null && isLeaderboardMetric(requestedMetric) ? requestedMetric : 'highestRound';
const options = { game: game || undefined, map: map || undefined, metric };
const profile = getPlayerProfile(params.guid, Number(url.searchParams.get('offset') ?? 0), options);
const { profile, entry } = await playerReport(params.guid, Number(url.searchParams.get('offset') ?? 0), options);
if (!profile) error(404, 'No recorded stats found for this player');
return { profile, returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')),
scope: { game: game ?? '', map: map ?? '', metric, entry: getPlayerLeaderboardEntry(params.guid, options) } };
scope: { game: game ?? '', map: map ?? '', metric, entry } };
}
+2 -2
View File
@@ -1,7 +1,7 @@
import { readMetrics } from '$lib/server/fleet/metrics.js';
import { metricsReport } from '$lib/server/reports/index.js';
import { parseMetricsQuery } from '$lib/metrics.js';
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = ({ url }) => {
return readMetrics(parseMetricsQuery(url.searchParams));
return metricsReport(parseMetricsQuery(url.searchParams));
};
@@ -0,0 +1,4 @@
import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
import { fleetOperations } from '$lib/server/fleet/overview.js';
export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }
+32 -16
View File
@@ -1,9 +1,13 @@
<script lang="ts">
let { data }: { data: import('./$types').PageData } = $props();
const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
import MatchHistory from '$ui/components/MatchHistory.svelte';
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
import { pollVisible } from '$ui/poll.js';
import { fetchCampaigns } from '$ui/campaigns.js';
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
import { serverTitle } from '$ui/server-labels.js';
@@ -105,9 +109,9 @@
}
let name = $derived($page.params.name ?? '');
let server = $derived(($fleetServers.find(s => s.name === name) as Server | undefined) ?? null);
let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null);
const native = $derived(!!server?.native_instance_id);
let integrations = $derived($fleetIntegrations);
let integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
let history = $state<DeploymentRow[]>([]);
let inventory = $state<InventoryEntry[]>([]);
let komodoProbe = $state<KomodoProbeResult | null>(null);
@@ -127,27 +131,39 @@
let tabLoading = $state(false);
let tabError = $state('');
let historyLimit = $state(25);
let historyTotal = $state(0);
let inventoryLimit = $state(25);
const operations = $derived($fleet.operations.filter(op => op.server === name));
const operations = $derived(snapshot.operations.filter(op => op.server === name));
async function loadTab(currentName: string, tab: string) {
let tabSequence = 0;
async function loadTab(currentName: string, tab: string, signal?: AbortSignal) {
if (tab !== 'history' && tab !== 'inventory') return;
const sequence = ++tabSequence;
try {
const response = await fetch(tab === 'history' ? '/api/campaigns?server=' + encodeURIComponent(currentName) : '/api/servers/' + encodeURIComponent(currentName) + '/scripts');
if (!response.ok) throw new Error('Could not refresh ' + tab + '.');
const body = await response.json();
if (name !== currentName || activeTab !== tab) return;
if (tab === 'history') history = body.deployments; else inventory = body.scripts;
if (tab === 'history') {
const body = await fetchCampaigns<DeploymentRow>({ server: currentName }, historyLimit, signal);
if (sequence !== tabSequence || name !== currentName || activeTab !== tab || signal?.aborted) return;
history = body.deployments; historyTotal = body.total;
} else {
const response = await fetch('/api/servers/' + encodeURIComponent(currentName) + '/scripts', { signal });
if (!response.ok) throw new Error('Could not refresh inventory.');
const body = await response.json();
if (sequence !== tabSequence || name !== currentName || activeTab !== tab || signal?.aborted) return;
inventory = body.scripts;
}
tabError = '';
} catch (error) { if (name === currentName && activeTab === tab) tabError = (error as Error).message; }
finally { if (name === currentName && activeTab === tab) tabLoading = false; }
return true;
} catch (error) {
if (sequence === tabSequence && name === currentName && activeTab === tab && !signal?.aborted) tabError = (error as Error).message;
return false;
} finally { if (sequence === tabSequence && name === currentName && activeTab === tab && !signal?.aborted) tabLoading = false; }
}
$effect(() => {
const currentName = name;
const tab = activeTab;
tabError = ''; history = []; inventory = []; historyLimit = 25; inventoryLimit = 25;
tabLoading = tab === 'history' || tab === 'inventory';
return pollVisible(() => loadTab(currentName, tab), 10000);
return pollVisible(signal => loadTab(currentName, tab, signal), 10000);
});
$effect(() => { name; komodoProbe = null; rconProbe = null; netDiagnosis = null; });
const fresh = $derived(server ? freshServer(server, $uiNow) : false);
@@ -312,7 +328,7 @@
</Card>
{#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={refreshFleet}>Retry</Button></div>{/if}
{#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={() => refreshFleet(true)}>Retry</Button></div>{/if}
{#if hasDiagnosticIssue}
<section class="connection-issue" aria-label="Server connection issue">
<div><strong>{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}</strong><p>Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : !native && integrations && !integrations.rcon ? 'RCON is not configured.' : ''}</p></div>
@@ -588,13 +604,13 @@
</div>
{/if}
</Card>
<ListFooter total={history.length} bind:limit={historyLimit} label="campaigns" />
<ListFooter total={historyTotal} bind:limit={historyLimit} onchange={() => loadTab(name, 'history')} label="campaigns" />
{/if}
</div>
</div>
{:else if $fleetError}
<div class="ui-error" role="alert">{$fleetError}<Button onclick={refreshFleet}>Retry</Button></div>
{:else if $fleet.receivedAt}
<div class="ui-error" role="alert">{$fleetError}<Button onclick={() => refreshFleet(true)}>Retry</Button></div>
{:else if $fleet.receivedAt || data.fleetSnapshot}
<EmptyState title="Server not found" description="This server may have been removed from the fleet." />
{:else}
<div class="loading">Loading…</div>