diff --git a/src/lib/server/fleet/overview.ts b/src/lib/server/fleet/overview.ts index df3cea9..dab98aa 100644 --- a/src/lib/server/fleet/overview.ts +++ b/src/lib/server/fleet/overview.ts @@ -2,19 +2,31 @@ import { actionLabels, type ProvisioningAction } from '../../provisioning.js'; import { db } from '../db/index.js'; import type { FleetOperation } from '../../fleet-overview.js'; -/** Read persisted work, including recent completions for one-shot lobby watches. */ +/** Fleet clients need only active work and unresolved failures. */ export function fleetOperations(now = Date.now()): FleetOperation[] { + return readOperations(now); +} + +/** Recent completions are loaded only by the server's own page. */ +export function serverOperations(server: string, now = Date.now()): FleetOperation[] { + return readOperations(now, server); +} + +function readOperations(now: number, server?: string): FleetOperation[] { const since = new Date(now - 86_400_000).toISOString(); + const bindings = { server: server ?? null, since }; + const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`; const handle = db(); const operations: FleetOperation[] = []; - const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL') - .all() as { server: string; requested_at: string; error: string | null }[]; + const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`) + .all(bindings) as { server: string; requested_at: string; error: string | null }[]; const recovering = new Map(slots.map(slot => [slot.server, slot])); const representedRecovery = new Set(); const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server, ds.restart_state, ds.error, ds.restarted_at, ds.players FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id - WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as { + WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done' + ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as { id: string; action: string; started_at: string; server: string; restart_state: string; error: string | null; restarted_at: string | null; players: number | null; }[]; @@ -34,19 +46,19 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { href: `/campaigns/${encodeURIComponent(row.id)}` }); } // Only the newest lifecycle operation can still require action on a server. - // Separate branches let SQLite use state and finish indexes; UNION deduplicates - // recent warnings that are also the newest operation. Unary + keeps terminal - // branches on finished_at and rowid instead of searching historical states. + // Separate branches use state/rowid indexes for fleet reads and the server + // index for detail history. Completed work is never loaded for the full fleet. const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments - WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health') + WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health') + ${server === undefined ? '' : `UNION + SELECT id, server, action, state, stage, error, started_at FROM server_deployments + WHERE server_deleted_at IS NULL AND server = @server + AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`} UNION SELECT id, server, action, state, stage, error, started_at FROM server_deployments - WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ? - UNION - SELECT id, server, action, state, stage, error, started_at FROM server_deployments - WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning') - AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`) - .all(since) as { id: string; server: string; action: string; state: string; stage: string; + WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted') + AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`) + .all(bindings) as { id: string; server: string; action: string; state: string; stage: string; error: string | null; started_at: string }[]; for (const row of lifecycle) { if (row.state === 'awaiting-health') representedRecovery.add(row.server); @@ -59,8 +71,8 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` }); } const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error - FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`) - .all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[]; + FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`) + .all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[]; const stages: Record = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta', 'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering', 'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' }; @@ -71,9 +83,9 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage, detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` }); } - const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed' + const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`) - .all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[]; + .all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[]; for (const job of maintenance) { operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance', server: job.server, state: 'attention', startedAt: job.requested_at, diff --git a/src/lib/server/fleet/snapshot.ts b/src/lib/server/fleet/snapshot.ts index f92ae74..9c63e76 100644 --- a/src/lib/server/fleet/snapshot.ts +++ b/src/lib/server/fleet/snapshot.ts @@ -1,26 +1,32 @@ import { db } from '../db/index.js'; -import { listServers } from './servers.js'; -import { observationReader } from './status.js'; +import { getServer, listServers } from './servers.js'; +import { observationReader, serverObservation } from './status.js'; import { config, komodoConfigured, rconConfigured } from '../config.js'; import { runningVersions } from '../integrations/plutonium.js'; import { listHosts } from '../hosts/index.js'; import type { NativeInstance } from '../backends/native.js'; +import { serverOperations } from './overview.js'; -export function fleetSnapshot() { +export function fleetSnapshot(serverName?: string) { + const bindings = serverName === undefined ? [] : [serverName]; const rconHost = config.rcon.host; const hosts = listHosts(); const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host])); - const versions = new Map(runningVersions().map(version => [version.server, version])); - const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance])); + const versions = new Map(runningVersions(serverName).map(version => [version.server, version])); + const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`) + .all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance])); const holds = new Map(); for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id - JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as + JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') + ${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as { server: string; target: string; betaId: number }[]) { const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries); } - const observe = observationReader(); - const servers = listServers().map((s) => { + const observe = serverName === undefined ? observationReader() : serverObservation; + const selected = serverName === undefined ? undefined : getServer(serverName); + const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : []; + const servers = rows.map((s) => { const instance = s.native_instance_id ? instances.get(s.name) : null; return { ...observe(s), @@ -36,7 +42,7 @@ export function fleetSnapshot() { return { servers, serverNow: Date.now(), - reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server), + reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [], integrations: { native: hosts.length > 0 || !komodoConfigured(), hostsReady: hosts.filter(host => host.readiness === 'ready').length, @@ -49,3 +55,6 @@ export function fleetSnapshot() { }; } +export function serverSnapshot(name: string) { + return { ...fleetSnapshot(name), operations: serverOperations(name) }; +} diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index 9c78380..a32e982 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -20,13 +20,18 @@ export function freshRelease(): number | null { const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN; return !latest.error && age >= 0 && age < HOUR ? latest.revision : null; } -export function runningVersions(): PlutoniumVersion[] { - const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[]; +export function runningVersions(serverName?: string): PlutoniumVersion[] { + const bindings = serverName === undefined ? [] : [serverName]; + const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`) + .all(...bindings) as PlutoniumVersion[]; const target = freshRelease(); const byServer = new Map(versions.map(version => [version.server, version])); - const epochs = latestGameSubmissions(); - const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server)); - return listServers().map(server => { + const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]); + const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`) + .all(...bindings) as { server: string }[]).map(row => row.server)); + const selected = serverName === undefined ? undefined : getServer(serverName); + const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : []; + return servers.map(server => { const value = byServer.get(server.name) ?? { server: server.name, revision: null, checked_at: null, error: null }; const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN; diff --git a/src/lib/ui/stores/fleet.ts b/src/lib/ui/stores/fleet.ts index baae9df..7be486c 100644 --- a/src/lib/ui/stores/fleet.ts +++ b/src/lib/ui/stores/fleet.ts @@ -6,156 +6,164 @@ import { browser } from '$app/environment'; import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js'; import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js'; -export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({ - servers: [], operations: [], receivedAt: 0 -}); -export const fleetError = writable(''); -export const fleetRefreshing = writable(false); -export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null); - -const observed = new Map(); -const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => { - const times = snapshot.servers.flatMap(server => { - const status = live.get(server.name); - return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt, - server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at, - server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt]; +// Each server detail view owns a scoped snapshot; it must not replace the shared fleet. +export function createFleetStore(serverName?: string) { + const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({ + servers: [], operations: [], receivedAt: 0 }); - return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(','); -}); -export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => { - const now = get(uiNow); - const next = snapshot.servers.map(server => { - const candidate = live.get(server.name); - const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined; - const match = matches.get(server.name); - const report = reports.get(server.name); - const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, - status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at, - server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt]; - const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(','); - const saved = observed.get(server.name); - if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value; - const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; }; - const base = status ? { ...server, last_players: status.players, last_map: status.map, - last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError, - roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince, - inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt, - game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state, - container_checked_at: status.containerCheckedAt ?? server.container_checked_at, - telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus, - telemetry: status.telemetry ?? server.telemetry } : server; - const current = selectServerObservation(base, reports.get(server.name), now); - // Apply lifecycle events immediately, until a newer status observes them - // or confirms that the lobby has emptied. - if (!match || current.health !== 'online' - || (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after)) - || Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '') - || match.receivedAt < (current.game_started_at ?? '') - || match.receivedAt < (current.game_ended_at ?? '')) return remember(current); - return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at, - game_ended_at: match.event === 'game_end' ? match.receivedAt : null }); + const fleetError = writable(''); + const fleetRefreshing = writable(false); + const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null); + + const observed = new Map(); + const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => { + const times = snapshot.servers.flatMap(server => { + const status = live.get(server.name); + return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt, + server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at, + server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt]; + }); + return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(','); }); - const names = new Set(snapshot.servers.map(server => server.name)); - for (const name of observed.keys()) if (!names.has(name)) observed.delete(name); - return next; -}); - - -let snapshotServerNow = 0; -export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable>; serverNow: number }) { - if (!browser || snapshot.serverNow < snapshotServerNow) return; - snapshotServerNow = snapshot.serverNow; - syncServerTime(snapshot.serverNow); - fleetIntegrations.set(snapshot.integrations); - fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) }); -} -function getIntegrations() { return get(fleetIntegrations); } - - - -let inFlight: Promise | null = null; -let refreshRequested = false; -export function refreshFleet(afterCurrent = false): Promise { - if (!browser) return Promise.resolve(); - if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; } - inFlight = (async () => { - fleetRefreshing.set(true); - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 12_000); - const requestedAt = Date.now(); - try { - const results = await Promise.allSettled([ - (async () => { - const response = await fetch('/api/servers', { signal: controller.signal }); - if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.'); - const servers = await response.json(); - if (servers.serverNow < snapshotServerNow) return; - snapshotServerNow = servers.serverNow; - syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations); - fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt })); - })(), - (async () => { - const response = await fetch('/api/fleet', { signal: controller.signal }); - if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.'); - const operations = await response.json(); - fleet.update(value => ({ ...value, operations: operations.operations })); - })() - ]); - const failure = results.find(result => result.status === 'rejected'); - if (failure?.status === 'rejected') throw failure.reason; - fleetError.set(''); - } catch (error) { - fleetError.set(error instanceof Error && error.name !== 'AbortError' - ? error.message : 'Fleet refresh timed out. Retrying automatically.'); - } finally { - clearTimeout(timeout); - fleetRefreshing.set(false); - } - })().finally(() => { - inFlight = null; - if (refreshRequested) { refreshRequested = false; void refreshFleet(); } + const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => { + const now = get(uiNow); + const next = snapshot.servers.map(server => { + const candidate = live.get(server.name); + const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined; + const match = matches.get(server.name); + const report = reports.get(server.name); + const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, + status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at, + server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt]; + const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(','); + const saved = observed.get(server.name); + if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value; + const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; }; + const base = status ? { ...server, last_players: status.players, last_map: status.map, + last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError, + roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince, + inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt, + game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state, + container_checked_at: status.containerCheckedAt ?? server.container_checked_at, + telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus, + telemetry: status.telemetry ?? server.telemetry } : server; + const current = selectServerObservation(base, reports.get(server.name), now); + // Apply lifecycle events immediately, until a newer status observes them + // or confirms that the lobby has emptied. + if (!match || current.health !== 'online' + || (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after)) + || Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '') + || match.receivedAt < (current.game_started_at ?? '') + || match.receivedAt < (current.game_ended_at ?? '')) return remember(current); + return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at, + game_ended_at: match.event === 'game_end' ? match.receivedAt : null }); + }); + const names = new Set(snapshot.servers.map(server => server.name)); + for (const name of observed.keys()) if (!names.has(name)) observed.delete(name); + return next; }); - return inFlight; -} -let stop: (() => void) | null = null; -export function startFleetMonitor(): () => void { - if (!browser || stop) return () => {}; - const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000); - let refreshTimer: ReturnType | undefined; - let stopped = false; - let pending = false; - let refreshing = false; - let lastEventRefresh = -Infinity; - const schedule = () => { - if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return; - refreshTimer = setTimeout(async () => { - refreshTimer = undefined; - refreshing = true; + + let snapshotServerNow = 0; + function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable>; serverNow: number }) { + if (!browser || snapshot.serverNow < snapshotServerNow) return; + snapshotServerNow = snapshot.serverNow; + syncServerTime(snapshot.serverNow); + fleetIntegrations.set(snapshot.integrations); + fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) }); + } + function getIntegrations() { return get(fleetIntegrations); } + + + + let inFlight: Promise | null = null; + let refreshRequested = false; + function refreshFleet(afterCurrent = false): Promise { + if (!browser) return Promise.resolve(); + if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; } + inFlight = (async () => { + fleetRefreshing.set(true); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 12_000); + const requestedAt = Date.now(); try { - // Wait for any manual/poll refresh, then fetch evidence newer than - // the event. Do not use refreshFleet(true)'s immediate follow-up. - if (inFlight) await inFlight; - if (stopped || document.visibilityState === 'hidden') return; - pending = false; - lastEventRefresh = Date.now(); - await refreshFleet(); + const results = await Promise.allSettled([ + (async () => { + const response = await fetch(serverName === undefined ? '/api/servers' : `/api/servers/${encodeURIComponent(serverName)}`, { signal: controller.signal }); + if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.'); + const servers = await response.json(); + if (servers.serverNow < snapshotServerNow) return; + snapshotServerNow = servers.serverNow; + syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations); + fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt, + operations: serverName === undefined ? value.operations : servers.operations })); + })(), + (async () => { + if (serverName !== undefined) return; + const response = await fetch('/api/fleet', { signal: controller.signal }); + if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.'); + const operations = await response.json(); + fleet.update(value => ({ ...value, operations: operations.operations })); + })() + ]); + const failure = results.find(result => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + fleetError.set(''); + } catch (error) { + fleetError.set(error instanceof Error && error.name !== 'AbortError' + ? error.message : 'Fleet refresh timed out. Retrying automatically.'); } finally { - refreshing = false; - schedule(); + clearTimeout(timeout); + fleetRefreshing.set(false); } - }, Math.max(250, lastEventRefresh + 2000 - Date.now())); - }; - const changed = () => { - if (document.visibilityState === 'hidden') return; - pending = true; - schedule(); - }; - window.addEventListener('ec:fleet-changed', changed); - stop = () => { - stopped = true; - stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null; - }; - return stop; + })().finally(() => { + inFlight = null; + if (refreshRequested) { refreshRequested = false; void refreshFleet(); } + }); + return inFlight; + } + + let stop: (() => void) | null = null; + function startFleetMonitor(): () => void { + if (!browser || stop) return () => {}; + const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000); + let refreshTimer: ReturnType | undefined; + let stopped = false; + let pending = false; + let refreshing = false; + let lastEventRefresh = -Infinity; + const schedule = () => { + if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return; + refreshTimer = setTimeout(async () => { + refreshTimer = undefined; + refreshing = true; + try { + // Wait for any manual/poll refresh, then fetch evidence newer than + // the event. Do not use refreshFleet(true)'s immediate follow-up. + if (inFlight) await inFlight; + if (stopped || document.visibilityState === 'hidden') return; + pending = false; + lastEventRefresh = Date.now(); + await refreshFleet(); + } finally { + refreshing = false; + schedule(); + } + }, Math.max(250, lastEventRefresh + 2000 - Date.now())); + }; + const changed = () => { + if (document.visibilityState === 'hidden') return; + pending = true; + schedule(); + }; + window.addEventListener('ec:fleet-changed', changed); + stop = () => { + stopped = true; + stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null; + }; + return stop; + } + return { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor }; } + +export const { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor } = createFleetStore(); diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 83aacf7..b33b882 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -75,7 +75,7 @@ $effect(() => { if (!sessionReady) return; const path = $page.url.pathname; - if (!(path === '/' || /^\/(servers|scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return; + if (!(path === '/' || /^\/(scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return; if ('fleetSnapshot' in $page.data && $page.data.fleetSnapshot) hydrateFleet($page.data.fleetSnapshot); return startFleetMonitor(); }); diff --git a/src/routes/api/servers/[name]/+server.ts b/src/routes/api/servers/[name]/+server.ts index 00587c7..578d5a2 100644 --- a/src/routes/api/servers/[name]/+server.ts +++ b/src/routes/api/servers/[name]/+server.ts @@ -1,6 +1,11 @@ import { kick } from '$lib/server/pipeline/worker.js'; import { json } from '@sveltejs/kit'; import { destroyServer } from '$lib/server/provisioning/index.js'; +import { serverSnapshot } from '$lib/server/fleet/snapshot.js'; + +export function GET({ params }: { params: { name: string } }) { + return json(serverSnapshot(params.name)); +} export async function DELETE({ params }: { params: { name: string } }) { const name = params.name; diff --git a/src/routes/servers/[name]/+page.server.ts b/src/routes/servers/[name]/+page.server.ts index 4d40e1f..dbaa2be 100644 --- a/src/routes/servers/[name]/+page.server.ts +++ b/src/routes/servers/[name]/+page.server.ts @@ -1,4 +1,5 @@ -import { fleetSnapshot } from '$lib/server/fleet/snapshot.js'; -import { fleetOperations } from '$lib/server/fleet/overview.js'; +import { serverSnapshot } from '$lib/server/fleet/snapshot.js'; -export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; } +export function load({ params }: { params: { name: string } }) { + return { serverSnapshot: serverSnapshot(params.name) }; +} diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index b9561f9..6057a8f 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -1,14 +1,12 @@