Fix false stale Fleet reports caused by clock skew

src/lib/ui/stores/fleet.ts

@@ -1,6 +1,6 @@
 import { derived, get, writable } from 'svelte/store';
 import { browser } from '$app/environment';
-import { statusByServer, statusReceivedAt } from './servers.js';
+import { statusByServer, statusReceivedAt, serverTimeNow, syncServerTime } from './servers.js';
 import { watchSatisfied, type FleetServer, type FleetOperation, type LobbyWatch } from '$lib/fleet-overview.js';
 import { serverTitle } from '../server-labels.js';
 import { toast } from '../toast.js';
@@ -39,6 +39,7 @@ export function refreshFleet(): Promise<void> {
             ]);
             if (responses.some(response => !response.ok)) throw new Error('Could not refresh Fleet. Retrying automatically.');
             const [servers, operations] = await Promise.all(responses.map(response => response.json()));
+            syncServerTime(servers.serverNow);
             fleetError.set('');
             fleetIntegrations.set(servers.integrations);
             fleet.set({ servers: servers.servers, operations: operations.operations, receivedAt: requestedAt });
@@ -93,7 +94,7 @@ export function startFleetMonitor(): () => void {
         if (!snapshot.receivedAt || Date.now() - snapshot.receivedAt > 30_000) return;
         const servers = get(fleetServers);
         for (const watch of get(lobbyWatches)) {
-            if (!watchSatisfied(watch, servers.find(server => server.name === watch.server), snapshot.operations, Date.now())) continue;
+            if (!watchSatisfied(watch, servers.find(server => server.name === watch.server), snapshot.operations, serverTimeNow())) continue;
             const message = `${serverTitle(watch.server)} ${watch.kind === 'empty' ? 'is empty' : watch.kind === 'online' ? 'is back online' : 'has finished its operation'}.`;
             cancelWatch(watch.id);
             toast(message, { variant: 'success', ttl: 0, href: watch.href });

src/lib/ui/stores/servers.ts

@@ -46,15 +46,36 @@ export const statusReceivedAt = writable(0);
 // startServerStream() call in the browser.
 const _uiNow = writable(Date.now());
 export const uiNow = readonly(_uiNow);
+let serverClock: { time: number; receivedAt: number } | null = null;
+
+// Report timestamps belong to the backend clock, which can differ from the
+// browser's wall clock. Advance that clock locally even if updates stop.
+export function serverTimeNow(): number {
+    return serverClock ? serverClock.time + performance.now() - serverClock.receivedAt : Date.now();
+}
+
+export function syncServerTime(serverNow?: number): void {
+    if (typeof serverNow === 'number' && Number.isFinite(serverNow)) {
+        // A delayed HTTP snapshot must not rewind time behind a newer stream
+        // update and make that update look like it came from the future.
+        const time = serverClock ? Math.max(serverNow, serverTimeNow()) : serverNow;
+        serverClock = { time, receivedAt: performance.now() };
+    }
+    // Refresh immediately: a suspended tab's interval may be well behind.
+    _uiNow.set(serverTimeNow());
+}
+
 let tickInterval: ReturnType<typeof setInterval> | null = null;
 
 let source: EventSource | null = null;
 
 export function startServerStream(): (() => void) | undefined {
     if (!browser || source) return;
+    syncServerTime();
     source = new EventSource('/api/events/servers');
     source.addEventListener('server-telemetry', (evt) => {
-        const data = JSON.parse((evt as MessageEvent).data) as { entries: GameTelemetry[]; snapshot?: boolean };
+        const data = JSON.parse((evt as MessageEvent).data) as { entries: GameTelemetry[]; snapshot?: boolean; serverNow?: number };
+        syncServerTime(data.serverNow);
         telemetryByServer.update((previous) => {
             const next = data.snapshot ? new Map<string, GameTelemetry>() : new Map(previous);
             for (const entry of data.entries) next.set(entry.server, entry);
@@ -62,7 +83,8 @@ export function startServerStream(): (() => void) | undefined {
         });
     });
     source.addEventListener('server-status', (evt) => {
-        const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[] };
+        const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number };
+        syncServerTime(data.serverNow);
         statusByServer.update(() => {
             const m = new Map<string, ServerStatus>();
             for (const e of data.entries) m.set(e.server, e);
@@ -71,7 +93,7 @@ export function startServerStream(): (() => void) | undefined {
         statusReceivedAt.set(Date.now());
     });
     if (tickInterval === null) {
-        tickInterval = setInterval(() => _uiNow.set(Date.now()), 1000);
+        tickInterval = setInterval(() => _uiNow.set(serverTimeNow()), 1000);
     }
     return () => { source?.close(); source = null; if (tickInterval) clearInterval(tickInterval); tickInterval = null; };
 }