Scope server snapshots and keep completed history off fleet refreshes
This commit is contained in:
@@ -2,19 +2,31 @@ import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
||||
import { db } from '../db/index.js';
|
||||
import type { FleetOperation } from '../../fleet-overview.js';
|
||||
|
||||
/** Read persisted work, including recent completions for one-shot lobby watches. */
|
||||
/** Fleet clients need only active work and unresolved failures. */
|
||||
export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
return readOperations(now);
|
||||
}
|
||||
|
||||
/** Recent completions are loaded only by the server's own page. */
|
||||
export function serverOperations(server: string, now = Date.now()): FleetOperation[] {
|
||||
return readOperations(now, server);
|
||||
}
|
||||
|
||||
function readOperations(now: number, server?: string): FleetOperation[] {
|
||||
const since = new Date(now - 86_400_000).toISOString();
|
||||
const bindings = { server: server ?? null, since };
|
||||
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
|
||||
const handle = db();
|
||||
const operations: FleetOperation[] = [];
|
||||
const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
|
||||
.all() as { server: string; requested_at: string; error: string | null }[];
|
||||
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
|
||||
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
|
||||
const recovering = new Map(slots.map(slot => [slot.server, slot]));
|
||||
const representedRecovery = new Set<string>();
|
||||
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
|
||||
ds.restart_state, ds.error, ds.restarted_at, ds.players
|
||||
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
|
||||
WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
|
||||
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
|
||||
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
|
||||
id: string; action: string; started_at: string; server: string; restart_state: string;
|
||||
error: string | null; restarted_at: string | null; players: number | null;
|
||||
}[];
|
||||
@@ -34,19 +46,19 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
href: `/campaigns/${encodeURIComponent(row.id)}` });
|
||||
}
|
||||
// Only the newest lifecycle operation can still require action on a server.
|
||||
// Separate branches let SQLite use state and finish indexes; UNION deduplicates
|
||||
// recent warnings that are also the newest operation. Unary + keeps terminal
|
||||
// branches on finished_at and rowid instead of searching historical states.
|
||||
// Separate branches use state/rowid indexes for fleet reads and the server
|
||||
// index for detail history. Completed work is never loaded for the full fleet.
|
||||
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
||||
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
||||
${server === undefined ? '' : `UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND server = @server
|
||||
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
|
||||
UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ?
|
||||
UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning')
|
||||
AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`)
|
||||
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
|
||||
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
|
||||
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
|
||||
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
|
||||
error: string | null; started_at: string }[];
|
||||
for (const row of lifecycle) {
|
||||
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
|
||||
@@ -59,8 +71,8 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
|
||||
}
|
||||
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
|
||||
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
|
||||
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
|
||||
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
|
||||
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
|
||||
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
|
||||
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
|
||||
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
|
||||
@@ -71,9 +83,9 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
|
||||
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
|
||||
}
|
||||
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
|
||||
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
|
||||
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
|
||||
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
|
||||
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
|
||||
for (const job of maintenance) {
|
||||
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
|
||||
server: job.server, state: 'attention', startedAt: job.requested_at,
|
||||
|
||||
@@ -1,26 +1,32 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { listServers } from './servers.js';
|
||||
import { observationReader } from './status.js';
|
||||
import { getServer, listServers } from './servers.js';
|
||||
import { observationReader, serverObservation } from './status.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { runningVersions } from '../integrations/plutonium.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import { serverOperations } from './overview.js';
|
||||
|
||||
export function fleetSnapshot() {
|
||||
export function fleetSnapshot(serverName?: string) {
|
||||
const bindings = serverName === undefined ? [] : [serverName];
|
||||
const rconHost = config.rcon.host;
|
||||
const hosts = listHosts();
|
||||
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
|
||||
const versions = new Map(runningVersions().map(version => [version.server, version]));
|
||||
const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
|
||||
const versions = new Map(runningVersions(serverName).map(version => [version.server, version]));
|
||||
const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
|
||||
.all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance]));
|
||||
const holds = new Map<string, { target: string; betaId: number }[]>();
|
||||
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
|
||||
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
|
||||
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
|
||||
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')
|
||||
${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as
|
||||
{ server: string; target: string; betaId: number }[]) {
|
||||
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
|
||||
}
|
||||
const observe = observationReader();
|
||||
const servers = listServers().map((s) => {
|
||||
const observe = serverName === undefined ? observationReader() : serverObservation;
|
||||
const selected = serverName === undefined ? undefined : getServer(serverName);
|
||||
const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
|
||||
const servers = rows.map((s) => {
|
||||
const instance = s.native_instance_id ? instances.get(s.name) : null;
|
||||
return {
|
||||
...observe(s),
|
||||
@@ -36,7 +42,7 @@ export function fleetSnapshot() {
|
||||
return {
|
||||
servers,
|
||||
serverNow: Date.now(),
|
||||
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
|
||||
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
|
||||
integrations: {
|
||||
native: hosts.length > 0 || !komodoConfigured(),
|
||||
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
|
||||
@@ -49,3 +55,6 @@ export function fleetSnapshot() {
|
||||
};
|
||||
}
|
||||
|
||||
export function serverSnapshot(name: string) {
|
||||
return { ...fleetSnapshot(name), operations: serverOperations(name) };
|
||||
}
|
||||
|
||||
@@ -20,13 +20,18 @@ export function freshRelease(): number | null {
|
||||
const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN;
|
||||
return !latest.error && age >= 0 && age < HOUR ? latest.revision : null;
|
||||
}
|
||||
export function runningVersions(): PlutoniumVersion[] {
|
||||
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
|
||||
export function runningVersions(serverName?: string): PlutoniumVersion[] {
|
||||
const bindings = serverName === undefined ? [] : [serverName];
|
||||
const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`)
|
||||
.all(...bindings) as PlutoniumVersion[];
|
||||
const target = freshRelease();
|
||||
const byServer = new Map(versions.map(version => [version.server, version]));
|
||||
const epochs = latestGameSubmissions();
|
||||
const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
|
||||
return listServers().map(server => {
|
||||
const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]);
|
||||
const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
|
||||
.all(...bindings) as { server: string }[]).map(row => row.server));
|
||||
const selected = serverName === undefined ? undefined : getServer(serverName);
|
||||
const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
|
||||
return servers.map(server => {
|
||||
const value = byServer.get(server.name)
|
||||
?? { server: server.name, revision: null, checked_at: null, error: null };
|
||||
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
|
||||
|
||||
+153
-145
@@ -6,156 +6,164 @@ import { browser } from '$app/environment';
|
||||
import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
|
||||
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
|
||||
|
||||
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
servers: [], operations: [], receivedAt: 0
|
||||
});
|
||||
export const fleetError = writable('');
|
||||
export const fleetRefreshing = writable(false);
|
||||
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
|
||||
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
|
||||
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
|
||||
const times = snapshot.servers.flatMap(server => {
|
||||
const status = live.get(server.name);
|
||||
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
|
||||
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
|
||||
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
|
||||
// Each server detail view owns a scoped snapshot; it must not replace the shared fleet.
|
||||
export function createFleetStore(serverName?: string) {
|
||||
const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
servers: [], operations: [], receivedAt: 0
|
||||
});
|
||||
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
});
|
||||
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const now = get(uiNow);
|
||||
const next = snapshot.servers.map(server => {
|
||||
const candidate = live.get(server.name);
|
||||
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
|
||||
const match = matches.get(server.name);
|
||||
const report = reports.get(server.name);
|
||||
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
|
||||
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
|
||||
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
|
||||
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
const saved = observed.get(server.name);
|
||||
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
|
||||
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
|
||||
const base = status ? { ...server, last_players: status.players, last_map: status.map,
|
||||
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
||||
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
||||
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
||||
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
|
||||
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
|
||||
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
|
||||
telemetry: status.telemetry ?? server.telemetry } : server;
|
||||
const current = selectServerObservation(base, reports.get(server.name), now);
|
||||
// Apply lifecycle events immediately, until a newer status observes them
|
||||
// or confirms that the lobby has emptied.
|
||||
if (!match || current.health !== 'online'
|
||||
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|
||||
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
||||
|| match.receivedAt < (current.game_started_at ?? '')
|
||||
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
|
||||
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
||||
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
|
||||
const fleetError = writable('');
|
||||
const fleetRefreshing = writable(false);
|
||||
const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
|
||||
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
|
||||
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
|
||||
const times = snapshot.servers.flatMap(server => {
|
||||
const status = live.get(server.name);
|
||||
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
|
||||
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
|
||||
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
|
||||
});
|
||||
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
});
|
||||
const names = new Set(snapshot.servers.map(server => server.name));
|
||||
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
|
||||
return next;
|
||||
});
|
||||
|
||||
|
||||
let snapshotServerNow = 0;
|
||||
export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
if (!browser || snapshot.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = snapshot.serverNow;
|
||||
syncServerTime(snapshot.serverNow);
|
||||
fleetIntegrations.set(snapshot.integrations);
|
||||
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
|
||||
}
|
||||
function getIntegrations() { return get(fleetIntegrations); }
|
||||
|
||||
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let refreshRequested = false;
|
||||
export function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
|
||||
inFlight = (async () => {
|
||||
fleetRefreshing.set(true);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 12_000);
|
||||
const requestedAt = Date.now();
|
||||
try {
|
||||
const results = await Promise.allSettled([
|
||||
(async () => {
|
||||
const response = await fetch('/api/servers', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
|
||||
const servers = await response.json();
|
||||
if (servers.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = servers.serverNow;
|
||||
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
|
||||
})(),
|
||||
(async () => {
|
||||
const response = await fetch('/api/fleet', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
|
||||
const operations = await response.json();
|
||||
fleet.update(value => ({ ...value, operations: operations.operations }));
|
||||
})()
|
||||
]);
|
||||
const failure = results.find(result => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
fleetError.set('');
|
||||
} catch (error) {
|
||||
fleetError.set(error instanceof Error && error.name !== 'AbortError'
|
||||
? error.message : 'Fleet refresh timed out. Retrying automatically.');
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
fleetRefreshing.set(false);
|
||||
}
|
||||
})().finally(() => {
|
||||
inFlight = null;
|
||||
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
|
||||
const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const now = get(uiNow);
|
||||
const next = snapshot.servers.map(server => {
|
||||
const candidate = live.get(server.name);
|
||||
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
|
||||
const match = matches.get(server.name);
|
||||
const report = reports.get(server.name);
|
||||
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
|
||||
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
|
||||
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
|
||||
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
const saved = observed.get(server.name);
|
||||
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
|
||||
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
|
||||
const base = status ? { ...server, last_players: status.players, last_map: status.map,
|
||||
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
||||
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
||||
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
||||
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
|
||||
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
|
||||
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
|
||||
telemetry: status.telemetry ?? server.telemetry } : server;
|
||||
const current = selectServerObservation(base, reports.get(server.name), now);
|
||||
// Apply lifecycle events immediately, until a newer status observes them
|
||||
// or confirms that the lobby has emptied.
|
||||
if (!match || current.health !== 'online'
|
||||
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|
||||
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
||||
|| match.receivedAt < (current.game_started_at ?? '')
|
||||
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
|
||||
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
||||
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
|
||||
});
|
||||
const names = new Set(snapshot.servers.map(server => server.name));
|
||||
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
|
||||
return next;
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
let stop: (() => void) | null = null;
|
||||
export function startFleetMonitor(): () => void {
|
||||
if (!browser || stop) return () => {};
|
||||
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let stopped = false;
|
||||
let pending = false;
|
||||
let refreshing = false;
|
||||
let lastEventRefresh = -Infinity;
|
||||
const schedule = () => {
|
||||
if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return;
|
||||
refreshTimer = setTimeout(async () => {
|
||||
refreshTimer = undefined;
|
||||
refreshing = true;
|
||||
|
||||
let snapshotServerNow = 0;
|
||||
function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
if (!browser || snapshot.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = snapshot.serverNow;
|
||||
syncServerTime(snapshot.serverNow);
|
||||
fleetIntegrations.set(snapshot.integrations);
|
||||
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
|
||||
}
|
||||
function getIntegrations() { return get(fleetIntegrations); }
|
||||
|
||||
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let refreshRequested = false;
|
||||
function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
|
||||
inFlight = (async () => {
|
||||
fleetRefreshing.set(true);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 12_000);
|
||||
const requestedAt = Date.now();
|
||||
try {
|
||||
// Wait for any manual/poll refresh, then fetch evidence newer than
|
||||
// the event. Do not use refreshFleet(true)'s immediate follow-up.
|
||||
if (inFlight) await inFlight;
|
||||
if (stopped || document.visibilityState === 'hidden') return;
|
||||
pending = false;
|
||||
lastEventRefresh = Date.now();
|
||||
await refreshFleet();
|
||||
const results = await Promise.allSettled([
|
||||
(async () => {
|
||||
const response = await fetch(serverName === undefined ? '/api/servers' : `/api/servers/${encodeURIComponent(serverName)}`, { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
|
||||
const servers = await response.json();
|
||||
if (servers.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = servers.serverNow;
|
||||
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt,
|
||||
operations: serverName === undefined ? value.operations : servers.operations }));
|
||||
})(),
|
||||
(async () => {
|
||||
if (serverName !== undefined) return;
|
||||
const response = await fetch('/api/fleet', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
|
||||
const operations = await response.json();
|
||||
fleet.update(value => ({ ...value, operations: operations.operations }));
|
||||
})()
|
||||
]);
|
||||
const failure = results.find(result => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
fleetError.set('');
|
||||
} catch (error) {
|
||||
fleetError.set(error instanceof Error && error.name !== 'AbortError'
|
||||
? error.message : 'Fleet refresh timed out. Retrying automatically.');
|
||||
} finally {
|
||||
refreshing = false;
|
||||
schedule();
|
||||
clearTimeout(timeout);
|
||||
fleetRefreshing.set(false);
|
||||
}
|
||||
}, Math.max(250, lastEventRefresh + 2000 - Date.now()));
|
||||
};
|
||||
const changed = () => {
|
||||
if (document.visibilityState === 'hidden') return;
|
||||
pending = true;
|
||||
schedule();
|
||||
};
|
||||
window.addEventListener('ec:fleet-changed', changed);
|
||||
stop = () => {
|
||||
stopped = true;
|
||||
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
|
||||
};
|
||||
return stop;
|
||||
})().finally(() => {
|
||||
inFlight = null;
|
||||
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
let stop: (() => void) | null = null;
|
||||
function startFleetMonitor(): () => void {
|
||||
if (!browser || stop) return () => {};
|
||||
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let stopped = false;
|
||||
let pending = false;
|
||||
let refreshing = false;
|
||||
let lastEventRefresh = -Infinity;
|
||||
const schedule = () => {
|
||||
if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return;
|
||||
refreshTimer = setTimeout(async () => {
|
||||
refreshTimer = undefined;
|
||||
refreshing = true;
|
||||
try {
|
||||
// Wait for any manual/poll refresh, then fetch evidence newer than
|
||||
// the event. Do not use refreshFleet(true)'s immediate follow-up.
|
||||
if (inFlight) await inFlight;
|
||||
if (stopped || document.visibilityState === 'hidden') return;
|
||||
pending = false;
|
||||
lastEventRefresh = Date.now();
|
||||
await refreshFleet();
|
||||
} finally {
|
||||
refreshing = false;
|
||||
schedule();
|
||||
}
|
||||
}, Math.max(250, lastEventRefresh + 2000 - Date.now()));
|
||||
};
|
||||
const changed = () => {
|
||||
if (document.visibilityState === 'hidden') return;
|
||||
pending = true;
|
||||
schedule();
|
||||
};
|
||||
window.addEventListener('ec:fleet-changed', changed);
|
||||
stop = () => {
|
||||
stopped = true;
|
||||
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
|
||||
};
|
||||
return stop;
|
||||
}
|
||||
return { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor };
|
||||
}
|
||||
|
||||
export const { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor } = createFleetStore();
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
$effect(() => {
|
||||
if (!sessionReady) return;
|
||||
const path = $page.url.pathname;
|
||||
if (!(path === '/' || /^\/(servers|scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
|
||||
if (!(path === '/' || /^\/(scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
|
||||
if ('fleetSnapshot' in $page.data && $page.data.fleetSnapshot) hydrateFleet($page.data.fleetSnapshot);
|
||||
return startFleetMonitor();
|
||||
});
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { kick } from '$lib/server/pipeline/worker.js';
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { destroyServer } from '$lib/server/provisioning/index.js';
|
||||
import { serverSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
|
||||
export function GET({ params }: { params: { name: string } }) {
|
||||
return json(serverSnapshot(params.name));
|
||||
}
|
||||
|
||||
export async function DELETE({ params }: { params: { name: string } }) {
|
||||
const name = params.name;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
import { fleetOperations } from '$lib/server/fleet/overview.js';
|
||||
import { serverSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
|
||||
export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }
|
||||
export function load({ params }: { params: { name: string } }) {
|
||||
return { serverSnapshot: serverSnapshot(params.name) };
|
||||
}
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
<script lang="ts">
|
||||
let { data }: { data: import('./$types').PageData } = $props();
|
||||
const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
|
||||
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
|
||||
import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
|
||||
import MatchHistory from '$ui/components/MatchHistory.svelte';
|
||||
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
|
||||
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { fetchCampaigns } from '$ui/campaigns.js';
|
||||
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
|
||||
import { createFleetStore } from '$ui/stores/fleet.js';
|
||||
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
|
||||
@@ -109,9 +107,17 @@
|
||||
}
|
||||
|
||||
let name = $derived($page.params.name ?? '');
|
||||
const detail = $derived(createFleetStore(name));
|
||||
const { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } = $derived(detail);
|
||||
$effect(() => {
|
||||
detail.hydrateFleet(data.serverSnapshot);
|
||||
return detail.startFleetMonitor();
|
||||
});
|
||||
const snapshot = $derived($fleet.receivedAt ? $fleet : data.serverSnapshot);
|
||||
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.serverSnapshot.servers);
|
||||
let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null);
|
||||
const native = $derived(!!server?.native_instance_id);
|
||||
let integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
|
||||
let integrations = $derived($fleetIntegrations ?? data.serverSnapshot.integrations);
|
||||
let history = $state<DeploymentRow[]>([]);
|
||||
let inventory = $state<InventoryEntry[]>([]);
|
||||
let komodoProbe = $state<KomodoProbeResult | null>(null);
|
||||
@@ -610,7 +616,7 @@
|
||||
</div>
|
||||
{:else if $fleetError}
|
||||
<div class="ui-error" role="alert">{$fleetError}<Button onclick={() => refreshFleet(true)}>Retry</Button></div>
|
||||
{:else if $fleet.receivedAt || data.fleetSnapshot}
|
||||
{:else if $fleet.receivedAt || data.serverSnapshot}
|
||||
<EmptyState title="Server not found" description="This server may have been removed from the fleet." />
|
||||
{:else}
|
||||
<div class="loading">Loading…</div>
|
||||
|
||||
Reference in New Issue
Block a user