Scope server snapshots and keep completed history off fleet refreshes
This commit is contained in:
+153
-145
@@ -6,156 +6,164 @@ import { browser } from '$app/environment';
|
||||
import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
|
||||
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
|
||||
|
||||
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
servers: [], operations: [], receivedAt: 0
|
||||
});
|
||||
export const fleetError = writable('');
|
||||
export const fleetRefreshing = writable(false);
|
||||
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
|
||||
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
|
||||
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
|
||||
const times = snapshot.servers.flatMap(server => {
|
||||
const status = live.get(server.name);
|
||||
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
|
||||
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
|
||||
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
|
||||
// Each server detail view owns a scoped snapshot; it must not replace the shared fleet.
|
||||
export function createFleetStore(serverName?: string) {
|
||||
const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
servers: [], operations: [], receivedAt: 0
|
||||
});
|
||||
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
});
|
||||
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const now = get(uiNow);
|
||||
const next = snapshot.servers.map(server => {
|
||||
const candidate = live.get(server.name);
|
||||
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
|
||||
const match = matches.get(server.name);
|
||||
const report = reports.get(server.name);
|
||||
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
|
||||
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
|
||||
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
|
||||
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
const saved = observed.get(server.name);
|
||||
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
|
||||
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
|
||||
const base = status ? { ...server, last_players: status.players, last_map: status.map,
|
||||
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
||||
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
||||
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
||||
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
|
||||
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
|
||||
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
|
||||
telemetry: status.telemetry ?? server.telemetry } : server;
|
||||
const current = selectServerObservation(base, reports.get(server.name), now);
|
||||
// Apply lifecycle events immediately, until a newer status observes them
|
||||
// or confirms that the lobby has emptied.
|
||||
if (!match || current.health !== 'online'
|
||||
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|
||||
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
||||
|| match.receivedAt < (current.game_started_at ?? '')
|
||||
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
|
||||
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
||||
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
|
||||
const fleetError = writable('');
|
||||
const fleetRefreshing = writable(false);
|
||||
const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
|
||||
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
|
||||
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
|
||||
const times = snapshot.servers.flatMap(server => {
|
||||
const status = live.get(server.name);
|
||||
return [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at, status?.telemetry?.receivedAt,
|
||||
server.last_seen_at, server.container_checked_at, server.last_checked_at, server.rcon_status?.last_seen_at,
|
||||
server.telemetry?.receivedAt, reports.get(server.name)?.receivedAt];
|
||||
});
|
||||
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
});
|
||||
const names = new Set(snapshot.servers.map(server => server.name));
|
||||
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
|
||||
return next;
|
||||
});
|
||||
|
||||
|
||||
let snapshotServerNow = 0;
|
||||
export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
if (!browser || snapshot.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = snapshot.serverNow;
|
||||
syncServerTime(snapshot.serverNow);
|
||||
fleetIntegrations.set(snapshot.integrations);
|
||||
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
|
||||
}
|
||||
function getIntegrations() { return get(fleetIntegrations); }
|
||||
|
||||
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let refreshRequested = false;
|
||||
export function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
|
||||
inFlight = (async () => {
|
||||
fleetRefreshing.set(true);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 12_000);
|
||||
const requestedAt = Date.now();
|
||||
try {
|
||||
const results = await Promise.allSettled([
|
||||
(async () => {
|
||||
const response = await fetch('/api/servers', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
|
||||
const servers = await response.json();
|
||||
if (servers.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = servers.serverNow;
|
||||
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
|
||||
})(),
|
||||
(async () => {
|
||||
const response = await fetch('/api/fleet', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
|
||||
const operations = await response.json();
|
||||
fleet.update(value => ({ ...value, operations: operations.operations }));
|
||||
})()
|
||||
]);
|
||||
const failure = results.find(result => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
fleetError.set('');
|
||||
} catch (error) {
|
||||
fleetError.set(error instanceof Error && error.name !== 'AbortError'
|
||||
? error.message : 'Fleet refresh timed out. Retrying automatically.');
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
fleetRefreshing.set(false);
|
||||
}
|
||||
})().finally(() => {
|
||||
inFlight = null;
|
||||
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
|
||||
const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const now = get(uiNow);
|
||||
const next = snapshot.servers.map(server => {
|
||||
const candidate = live.get(server.name);
|
||||
const status = (candidate?.receivedAt ?? received) > snapshot.receivedAt ? candidate : undefined;
|
||||
const match = matches.get(server.name);
|
||||
const report = reports.get(server.name);
|
||||
const times = [status?.lastSeenAt, status?.containerCheckedAt, status?.rconStatus?.last_seen_at,
|
||||
status?.telemetry?.receivedAt, server.last_seen_at, server.container_checked_at, server.last_checked_at,
|
||||
server.rcon_status?.last_seen_at, server.telemetry?.receivedAt, report?.receivedAt];
|
||||
const phase = times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
const saved = observed.get(server.name);
|
||||
if (saved && saved.server === server && saved.status === status && saved.match === match && saved.report === report && saved.phase === phase) return saved.value;
|
||||
const remember = (value: FleetServer) => { observed.set(server.name, { server, status, match, report, phase, value }); return value; };
|
||||
const base = status ? { ...server, last_players: status.players, last_map: status.map,
|
||||
last_gametype: status.gametype, last_seen_at: status.lastSeenAt, last_error: status.lastError,
|
||||
roster: status.roster, health: status.health, endpoint: status.endpoint, replySnippet: status.replySnippet, active_since: status.activeSince,
|
||||
inactive_since: status.inactiveSince, game_started_at: status.gameStartedAt,
|
||||
game_ended_at: status.gameEndedAt, container_state: status.containerState ?? server.container_state,
|
||||
container_checked_at: status.containerCheckedAt ?? server.container_checked_at,
|
||||
telemetry_after: status.telemetryAfter ?? server.telemetry_after, rcon_status: status.rconStatus,
|
||||
telemetry: status.telemetry ?? server.telemetry } : server;
|
||||
const current = selectServerObservation(base, reports.get(server.name), now);
|
||||
// Apply lifecycle events immediately, until a newer status observes them
|
||||
// or confirms that the lobby has emptied.
|
||||
if (!match || current.health !== 'online'
|
||||
|| (current.telemetry_after && Date.parse(match.receivedAt) <= Date.parse(current.telemetry_after))
|
||||
|| Date.parse(match.receivedAt) <= Date.parse(current.last_seen_at ?? '')
|
||||
|| match.receivedAt < (current.game_started_at ?? '')
|
||||
|| match.receivedAt < (current.game_ended_at ?? '')) return remember(current);
|
||||
return remember({ ...current, game_started_at: match.event === 'game_start' ? match.receivedAt : current.game_started_at,
|
||||
game_ended_at: match.event === 'game_end' ? match.receivedAt : null });
|
||||
});
|
||||
const names = new Set(snapshot.servers.map(server => server.name));
|
||||
for (const name of observed.keys()) if (!names.has(name)) observed.delete(name);
|
||||
return next;
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
let stop: (() => void) | null = null;
|
||||
export function startFleetMonitor(): () => void {
|
||||
if (!browser || stop) return () => {};
|
||||
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let stopped = false;
|
||||
let pending = false;
|
||||
let refreshing = false;
|
||||
let lastEventRefresh = -Infinity;
|
||||
const schedule = () => {
|
||||
if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return;
|
||||
refreshTimer = setTimeout(async () => {
|
||||
refreshTimer = undefined;
|
||||
refreshing = true;
|
||||
|
||||
let snapshotServerNow = 0;
|
||||
function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
if (!browser || snapshot.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = snapshot.serverNow;
|
||||
syncServerTime(snapshot.serverNow);
|
||||
fleetIntegrations.set(snapshot.integrations);
|
||||
fleet.set({ servers: snapshot.servers, operations: snapshot.operations, receivedAt: Date.now() - Math.max(0, serverTimeNow() - snapshot.serverNow) });
|
||||
}
|
||||
function getIntegrations() { return get(fleetIntegrations); }
|
||||
|
||||
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let refreshRequested = false;
|
||||
function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
|
||||
inFlight = (async () => {
|
||||
fleetRefreshing.set(true);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 12_000);
|
||||
const requestedAt = Date.now();
|
||||
try {
|
||||
// Wait for any manual/poll refresh, then fetch evidence newer than
|
||||
// the event. Do not use refreshFleet(true)'s immediate follow-up.
|
||||
if (inFlight) await inFlight;
|
||||
if (stopped || document.visibilityState === 'hidden') return;
|
||||
pending = false;
|
||||
lastEventRefresh = Date.now();
|
||||
await refreshFleet();
|
||||
const results = await Promise.allSettled([
|
||||
(async () => {
|
||||
const response = await fetch(serverName === undefined ? '/api/servers' : `/api/servers/${encodeURIComponent(serverName)}`, { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
|
||||
const servers = await response.json();
|
||||
if (servers.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = servers.serverNow;
|
||||
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt,
|
||||
operations: serverName === undefined ? value.operations : servers.operations }));
|
||||
})(),
|
||||
(async () => {
|
||||
if (serverName !== undefined) return;
|
||||
const response = await fetch('/api/fleet', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
|
||||
const operations = await response.json();
|
||||
fleet.update(value => ({ ...value, operations: operations.operations }));
|
||||
})()
|
||||
]);
|
||||
const failure = results.find(result => result.status === 'rejected');
|
||||
if (failure?.status === 'rejected') throw failure.reason;
|
||||
fleetError.set('');
|
||||
} catch (error) {
|
||||
fleetError.set(error instanceof Error && error.name !== 'AbortError'
|
||||
? error.message : 'Fleet refresh timed out. Retrying automatically.');
|
||||
} finally {
|
||||
refreshing = false;
|
||||
schedule();
|
||||
clearTimeout(timeout);
|
||||
fleetRefreshing.set(false);
|
||||
}
|
||||
}, Math.max(250, lastEventRefresh + 2000 - Date.now()));
|
||||
};
|
||||
const changed = () => {
|
||||
if (document.visibilityState === 'hidden') return;
|
||||
pending = true;
|
||||
schedule();
|
||||
};
|
||||
window.addEventListener('ec:fleet-changed', changed);
|
||||
stop = () => {
|
||||
stopped = true;
|
||||
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
|
||||
};
|
||||
return stop;
|
||||
})().finally(() => {
|
||||
inFlight = null;
|
||||
if (refreshRequested) { refreshRequested = false; void refreshFleet(); }
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
let stop: (() => void) | null = null;
|
||||
function startFleetMonitor(): () => void {
|
||||
if (!browser || stop) return () => {};
|
||||
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let stopped = false;
|
||||
let pending = false;
|
||||
let refreshing = false;
|
||||
let lastEventRefresh = -Infinity;
|
||||
const schedule = () => {
|
||||
if (stopped || refreshing || refreshTimer !== undefined || !pending || document.visibilityState === 'hidden') return;
|
||||
refreshTimer = setTimeout(async () => {
|
||||
refreshTimer = undefined;
|
||||
refreshing = true;
|
||||
try {
|
||||
// Wait for any manual/poll refresh, then fetch evidence newer than
|
||||
// the event. Do not use refreshFleet(true)'s immediate follow-up.
|
||||
if (inFlight) await inFlight;
|
||||
if (stopped || document.visibilityState === 'hidden') return;
|
||||
pending = false;
|
||||
lastEventRefresh = Date.now();
|
||||
await refreshFleet();
|
||||
} finally {
|
||||
refreshing = false;
|
||||
schedule();
|
||||
}
|
||||
}, Math.max(250, lastEventRefresh + 2000 - Date.now()));
|
||||
};
|
||||
const changed = () => {
|
||||
if (document.visibilityState === 'hidden') return;
|
||||
pending = true;
|
||||
schedule();
|
||||
};
|
||||
window.addEventListener('ec:fleet-changed', changed);
|
||||
stop = () => {
|
||||
stopped = true;
|
||||
stopPolling(); clearTimeout(refreshTimer); window.removeEventListener('ec:fleet-changed', changed); stop = null;
|
||||
};
|
||||
return stop;
|
||||
}
|
||||
return { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor };
|
||||
}
|
||||
|
||||
export const { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor } = createFleetStore();
|
||||
|
||||
Reference in New Issue
Block a user