josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix false stale Fleet reports caused by clock skew

Browse Source
This commit is contained in:
josh
2026-09-19 20:17:43 -04:00
parent 345839fbd4
commit 1d073779fe
9 changed files with 224 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/ui/components/ServerCard.svelte
+4 -4
View File
@@ -54,9 +54,9 @@
mapLabelOverride
}: Props = $props();
function ago(iso: string | null): string {
function ago(iso: string | null, now: number): string {
if (!iso) return 'never';
const d = Date.now() - new Date(iso).getTime();
const d = now - new Date(iso).getTime();
if (d < 60_000) return `${Math.max(1, Math.floor(d / 1000))}s ago`;
if (d < 3_600_000) return `${Math.floor(d / 60_000)}m ago`;
if (d < 86_400_000) return `${Math.floor(d / 3_600_000)}h ago`;
@@ -175,9 +175,9 @@
{#if health === 'offline' && lastError}
<span class="err">{lastError}</span>
{:else if lastSeenAt}
<span>Last seen {ago(lastSeenAt)}</span>
<span>Last seen {ago(lastSeenAt, $uiNow)}</span>
{:else if lastRestartedAt}
<span>Restart {ago(lastRestartedAt)}</span>
<span>Restart {ago(lastRestartedAt, $uiNow)}</span>
{:else}
<span class="dim">No data</span>
{/if}
src/lib/ui/stores/fleet.ts
+3 -2
View File
@@ -1,6 +1,6 @@
import { derived, get, writable } from 'svelte/store';
import { browser } from '$app/environment';
import { statusByServer, statusReceivedAt } from './servers.js';
import { statusByServer, statusReceivedAt, serverTimeNow, syncServerTime } from './servers.js';
import { watchSatisfied, type FleetServer, type FleetOperation, type LobbyWatch } from '$lib/fleet-overview.js';
import { serverTitle } from '../server-labels.js';
import { toast } from '../toast.js';
@@ -39,6 +39,7 @@ export function refreshFleet(): Promise<void> {
]);
if (responses.some(response => !response.ok)) throw new Error('Could not refresh Fleet. Retrying automatically.');
const [servers, operations] = await Promise.all(responses.map(response => response.json()));
syncServerTime(servers.serverNow);
fleetError.set('');
fleetIntegrations.set(servers.integrations);
fleet.set({ servers: servers.servers, operations: operations.operations, receivedAt: requestedAt });
@@ -93,7 +94,7 @@ export function startFleetMonitor(): () => void {
if (!snapshot.receivedAt || Date.now() - snapshot.receivedAt > 30_000) return;
const servers = get(fleetServers);
for (const watch of get(lobbyWatches)) {
if (!watchSatisfied(watch, servers.find(server => server.name === watch.server), snapshot.operations, Date.now())) continue;
if (!watchSatisfied(watch, servers.find(server => server.name === watch.server), snapshot.operations, serverTimeNow())) continue;
const message = `${serverTitle(watch.server)} ${watch.kind === 'empty' ? 'is empty' : watch.kind === 'online' ? 'is back online' : 'has finished its operation'}.`;
cancelWatch(watch.id);
toast(message, { variant: 'success', ttl: 0, href: watch.href });
src/lib/ui/stores/servers.ts
+25 -3
View File
@@ -46,15 +46,36 @@ export const statusReceivedAt = writable(0);
// startServerStream() call in the browser.
const _uiNow = writable(Date.now());
export const uiNow = readonly(_uiNow);
let serverClock: { time: number; receivedAt: number } | null = null;
// Report timestamps belong to the backend clock, which can differ from the
// browser's wall clock. Advance that clock locally even if updates stop.
export function serverTimeNow(): number {
return serverClock ? serverClock.time + performance.now() - serverClock.receivedAt : Date.now();
}
export function syncServerTime(serverNow?: number): void {
if (typeof serverNow === 'number' && Number.isFinite(serverNow)) {
// A delayed HTTP snapshot must not rewind time behind a newer stream
// update and make that update look like it came from the future.
const time = serverClock ? Math.max(serverNow, serverTimeNow()) : serverNow;
serverClock = { time, receivedAt: performance.now() };
}
// Refresh immediately: a suspended tab's interval may be well behind.
_uiNow.set(serverTimeNow());
}
let tickInterval: ReturnType<typeof setInterval> | null = null;
let source: EventSource | null = null;
export function startServerStream(): (() => void) | undefined {
if (!browser || source) return;
syncServerTime();
source = new EventSource('/api/events/servers');
source.addEventListener('server-telemetry', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { entries: GameTelemetry[]; snapshot?: boolean };
const data = JSON.parse((evt as MessageEvent).data) as { entries: GameTelemetry[]; snapshot?: boolean; serverNow?: number };
syncServerTime(data.serverNow);
telemetryByServer.update((previous) => {
const next = data.snapshot ? new Map<string, GameTelemetry>() : new Map(previous);
for (const entry of data.entries) next.set(entry.server, entry);
@@ -62,7 +83,8 @@ export function startServerStream(): (() => void) | undefined {
});
});
source.addEventListener('server-status', (evt) => {
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[] };
const data = JSON.parse((evt as MessageEvent).data) as { entries: ServerStatus[]; serverNow?: number };
syncServerTime(data.serverNow);
statusByServer.update(() => {
const m = new Map<string, ServerStatus>();
for (const e of data.entries) m.set(e.server, e);
@@ -71,7 +93,7 @@ export function startServerStream(): (() => void) | undefined {
statusReceivedAt.set(Date.now());
});
if (tickInterval === null) {
tickInterval = setInterval(() => _uiNow.set(Date.now()), 1000);
tickInterval = setInterval(() => _uiNow.set(serverTimeNow()), 1000);
}
return () => { source?.close(); source = null; if (tickInterval) clearInterval(tickInterval); tickInterval = null; };
}
src/routes/api/events/servers/+server.ts
+2 -2
View File
@@ -12,11 +12,11 @@ export async function GET() {
start(controller) {
unsubscribe = subscribe((event) => {
if (event.type === 'server-status' || event.type === 'server-telemetry') {
controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`));
controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, serverNow: Date.now() })}\n\n`));
}
});
controller.enqueue(encoder.encode(': hello\n\n'));
controller.enqueue(encoder.encode(`event: server-telemetry\ndata: ${JSON.stringify({ entries: listTelemetry(), snapshot: true })}\n\n`));
controller.enqueue(encoder.encode(`event: server-telemetry\ndata: ${JSON.stringify({ entries: listTelemetry(), snapshot: true, serverNow: Date.now() })}\n\n`));
heartbeat = setInterval(() => controller.enqueue(encoder.encode(': ping\n\n')), HEARTBEAT_MS);
},
cancel() {
src/routes/api/servers/+server.ts
+1
View File
@@ -40,6 +40,7 @@ export async function GET() {
});
return json({
servers,
serverNow: Date.now(),
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
integrations: {
rcon: rconConfigured(),
src/routes/servers/[name]/+page.svelte
+4 -4
View File
@@ -154,9 +154,9 @@
let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting');
function ago(iso: string | null): string {
function ago(iso: string | null, now: number): string {
if (!iso) return 'never';
const d = Date.now() - new Date(iso).getTime();
const d = now - new Date(iso).getTime();
if (d < 60_000) return `${Math.max(1, Math.floor(d / 1000))}s ago`;
if (d < 3_600_000) return `${Math.floor(d / 60_000)}m ago`;
if (d < 86_400_000) return `${Math.floor(d / 3_600_000)}h ago`;
@@ -270,7 +270,7 @@
{/if}
{#if effectiveHealth !== 'online'}
<div class="metric-cell">
<Stat label="Last seen" value={ago(effectiveSeen)} size="md" tween={false} />
<Stat label="Last seen" value={ago(effectiveSeen, $uiNow)} size="md" tween={false} />
</div>
{/if}
</div>
@@ -517,7 +517,7 @@
{/if}
</span>
<span class="hist-files">{mine.files_changed} file{mine.files_changed === 1 ? '' : 's'}</span>
<span class="hist-when">{ago(d.started_at)}</span>
<span class="hist-when">{ago(d.started_at, $uiNow)}</span>
{#if mine.error}<span class="hist-err">{mine.error}</span>{/if}
</a>
{/each}