feat: add Watchdog recovery and drift rebuild workflows

src/lib/server/fleet/status-evidence.ts

@@ -0,0 +1,46 @@
+import { db } from '../db/index.js';
+import { parseGameEvent } from '../../telemetry.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+
+export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
+
+/** Only validated, recently received reports can establish current game state. */
+export function telemetryEvidence(server: string, after?: string | null): TelemetryEvidence | null {
+    const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
+        { received_at: string; payload: string | null } | undefined;
+    if (!row?.payload) return null;
+    const received = Date.parse(row.received_at);
+    const age = Date.now() - received;
+    if (!Number.isFinite(age) || age < 0 || age > ONLINE_MAX_MS) return null;
+    if (after && (!Number.isFinite(Date.parse(after)) || received <= Date.parse(after))) return null;
+    try {
+        const event = parseGameEvent(JSON.parse(row.payload));
+        if (event.server !== server) return null;
+        return { players: event.players.length, map: event.map, receivedAt: row.received_at };
+    } catch { return null; }
+}
+
+/** Positive occupancy from either source always vetoes an empty-server claim. */
+export async function probePlayerCount(server: string): Promise<number | null> {
+    const players = await rconGetPlayerCount(server).catch(() => null);
+    // Read after the network request: a player may join while RCON is pending.
+    const telemetry = telemetryEvidence(server);
+    if (players !== null && players > 0) return players;
+    if (telemetry && telemetry.players > 0) return telemetry.players;
+    return players === 0 || telemetry?.players === 0 ? 0 : null;
+}
+
+/** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
+export async function probeServerStatus(server: string, after?: string | null): Promise<
+    { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
+> {
+    let reason = 'RCON unavailable';
+    try {
+        const rcon = await rconStatusResult(server);
+        if (rcon.ok) return { ok: true, players: Math.max(rcon.status.players, telemetryEvidence(server, after)?.players ?? 0), map: rcon.status.map, source: 'rcon' };
+        reason = rcon.reason;
+    } catch (err) { reason = (err as Error).message; }
+    const telemetry = telemetryEvidence(server, after);
+    return telemetry ? { ok: true, ...telemetry, source: 'telemetry' } : { ok: false, reason };
+}