Fix native health polling and isolate file operation traffic
This commit is contained in:
@@ -16,6 +16,7 @@ import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident
|
||||
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
|
||||
|
||||
export const WATCHDOG_SCAN_MS = 60_000;
|
||||
export const WATCHDOG_DRIFT_SCAN_MS = 5 * 60_000;
|
||||
export const WATCHDOG_FAILURE_MS = 120_000;
|
||||
export const WATCHDOG_RECOVERY_MS = 300_000;
|
||||
const iso = () => new Date().toISOString();
|
||||
@@ -240,7 +241,7 @@ async function observe(server: string, operationId?: string): Promise<Observatio
|
||||
return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
|
||||
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
|
||||
}
|
||||
async function scanServer(server: string): Promise<void> {
|
||||
async function scanServer(server: string, forceScan: boolean): Promise<void> {
|
||||
const row = getServer(server);
|
||||
if (!row || row.desired_state === 'stopped') return;
|
||||
if (scanSuppressed(server) || protectedWork(server)) return;
|
||||
@@ -265,8 +266,19 @@ async function scanServer(server: string): Promise<void> {
|
||||
}
|
||||
// Unknown container health is an observation failure, not proof that a rebuild is safe.
|
||||
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
|
||||
// File trees change far less often than liveness. Persist the last successful
|
||||
// scan per registration/process so health checks do not rescan every minute.
|
||||
// Manual checks and new lifecycle submissions always get fresh file evidence.
|
||||
const driftKey = `watchdog_drift_scan:${server}`;
|
||||
const instance = nativeInstance(server);
|
||||
const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]);
|
||||
let previous: { identity?: string; at?: string } = {};
|
||||
try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ }
|
||||
const age = Date.now() - Date.parse(previous?.at ?? '');
|
||||
if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return;
|
||||
const prepared = await prepareDriftCampaign(server);
|
||||
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
|
||||
setSetting(driftKey, JSON.stringify({ identity, at: iso() }));
|
||||
if (!prepared.plan.files.length) {
|
||||
const current = active(server, 'drift');
|
||||
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
|
||||
@@ -305,7 +317,7 @@ async function run(forceScan: boolean): Promise<void> {
|
||||
async function pump() {
|
||||
while (cursor < servers.length) {
|
||||
const server = servers[cursor++];
|
||||
try { await scanServer(server.name); }
|
||||
try { await scanServer(server.name, forceScan); }
|
||||
catch (err) {
|
||||
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user