diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 872eddb..3079a19 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.3" +VERSION = "1.4.5" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -1092,7 +1092,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1126,11 +1126,20 @@ class Agent: bind = saved.get("bindAddress", "0.0.0.0") host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind family = socket.AF_INET6 if ":" in host else socket.AF_INET + # Retry only the read-only occupancy query. A missing UDP reply must + # never cause commands with effects to be replayed. + for attempt in range(3 if command == "status" else 1): + result = self.rcon_attempt(family, host, int(saved["port"]), password, command) + if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status": + return result + time.sleep(0.25 * 2 ** attempt) + + def rcon_attempt(self, family, host, port, password, command): chunks = [] with socket.socket(family, socket.SOCK_DGRAM) as sock: sock.settimeout(3) try: - sock.connect((host, int(saved["port"]))) + sock.connect((host, port)) sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n") while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024: chunks.append(sock.recv(65535)) @@ -2016,7 +2025,11 @@ class Agent: key = operation.get("instanceId") return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext() - def read_loop(self): + @staticmethod + def health_operation(operation): + return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version")) + + def read_loop(self, lane="read"): # Each lane owns its SQLite connection and never writes executing_id, # checkpoints, or current_operation used by the mutation state machine. journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False) @@ -2026,10 +2039,13 @@ class Agent: if self.update_pending(): self.stopping.wait(3) continue - response = self.request("GET", "/api/agent/operations?lane=read&wait=20") + response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20") + handled = False for operation in response.get("operations", []): if self.stopping.is_set() or not self.read_operation(operation): continue + if lane == "health" and not self.health_operation(operation): + continue with self.read_claim_lock: if operation["id"] in self.read_claims: continue @@ -2038,19 +2054,26 @@ class Agent: with self.activity(read_only=True) as acquired: if not acquired or self.update_pending(): continue - with self.read_gate.read(), self.instance_lock(operation): - row, _ = journal.accept(operation) - if row["state"] in ("accepted", "unknown", "running"): - self.report(journal.update(operation["id"], "running")) + row, _ = journal.accept(operation) + if row["state"] in ("accepted", "unknown", "running"): + self.report(journal.update(operation["id"], "running")) + with self.read_gate.read(), self.instance_lock(operation): try: result = self.dispatch(operation) row = journal.update(operation["id"], "completed", result) except Exception as err: row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable") - self.report(row) + # Receipt delivery can take seconds; it must not + # hold the instance lock needed by health checks. + self.report(row) + handled = True finally: with self.read_claim_lock: self.read_claims.discard(operation["id"]) + if response.get("operations") and not handled: + # A legacy controller may return another lane's work or + # a read already claimed locally. Avoid a busy poll loop. + self.stopping.wait(0.25) except (AgentError, OSError, ValueError, KeyError, sqlite3.Error): self.stopping.wait(3) finally: @@ -2109,7 +2132,8 @@ class Agent: self.stopping.wait(3) heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True) heartbeats.start() - readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)] + # Preserve two concurrent health checks even while files are being read. + readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")] for reader in readers: reader.start() while not self.stopping.is_set(): diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 0c4a4d5..dab0e0a 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import threading import unittest -from unittest.mock import mock_open, patch +from unittest.mock import MagicMock, call, mock_open, patch spec = importlib.util.spec_from_file_location("ec_agent", Path(__file__).resolve().parents[1] / "ec_agent.py") module = importlib.util.module_from_spec(spec) @@ -995,6 +995,64 @@ class AgentTests(unittest.TestCase): self.assertEqual(result["players"], 1) self.assertTrue(result["ok"]) + @staticmethod + def rcon_socket(*replies): + sock = MagicMock() + sock.__enter__.return_value = sock + sock.recv.side_effect = replies + return sock + + def test_udp_status_recovers_after_transient_failures(self): + self.create() + for error in (module.socket.timeout(), OSError("unreachable")): + with self.subTest(error=type(error).__name__): + failed = self.rcon_socket(error) + # Keep all response packets, including occupied player rows. + recovered = self.rcon_socket( + b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n", + b"\xff\xff\xff\xffprint\n0 42 CNCT abc player\n", module.socket.timeout()) + with patch.object(module.socket, "socket", side_effect=[failed, recovered]) as sockets, patch.object(module.time, "sleep") as sleep: + result = self.agent.rcon(INSTANCE, 1, "status") + self.assertTrue(result["ok"]) + self.assertEqual(result["players"], 1) + self.assertEqual(sockets.call_count, 2) + sleep.assert_called_once_with(0.25) + failed.__exit__.assert_called_once() + recovered.__exit__.assert_called_once() + + def test_udp_status_retries_are_bounded(self): + self.create() + attempts = [self.rcon_socket(module.socket.timeout()) for _ in range(3)] + with patch.object(module.socket, "socket", side_effect=attempts) as sockets, patch.object(module.time, "sleep") as sleep: + result = self.agent.rcon(INSTANCE, 1, "status") + self.assertFalse(result["ok"]) + self.assertEqual(result["reason"], "timeout") + self.assertEqual(sockets.call_count, 3) + self.assertEqual(sleep.call_args_list, [call(0.25), call(0.5)]) + + def test_udp_status_does_not_retry_bad_replies_or_partial_tables(self): + self.create() + for packet in (b"wrong listener", b"\xff\xff\xff\xffprint\nBad rcon_password", + b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n"): + with self.subTest(packet=packet): + sock = self.rcon_socket(packet, module.socket.timeout()) + with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep: + result = self.agent.rcon(INSTANCE, 1, "status") + self.assertIsNone(result["players"]) + sockets.assert_called_once() + sleep.assert_not_called() + + def test_udp_commands_with_effects_are_never_replayed(self): + self.create() + for command in ("map_restart", "kick 0", "version"): + with self.subTest(command=command): + sock = self.rcon_socket(module.socket.timeout()) + with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep: + result = self.agent.rcon(INSTANCE, 1, command) + self.assertEqual(result["reason"], "timeout") + sockets.assert_called_once() + sleep.assert_not_called() + def test_t6_bot_column_allows_empty_restart_but_holds_players_and_partial_replies(self): self.create() header = "map: zm_nuked\nnum score bot ping guid name lastmsg address qport rate\n" diff --git a/agent/tests/test_update_integration.py b/agent/tests/test_update_integration.py index 4228ce9..a4a744a 100644 --- a/agent/tests/test_update_integration.py +++ b/agent/tests/test_update_integration.py @@ -118,6 +118,44 @@ class UpdateIntegrationTests(unittest.TestCase): self.agent.read_loop() dispatch.assert_not_called() + def test_health_lane_runs_while_file_receipt_for_same_instance_is_slow(self): + receipt_waiting, release_receipt, health_done = threading.Event(), threading.Event(), threading.Event() + file_op = {"id": "file-one", "kind": "files.read", "instanceId": "same", "payload": {}} + health_op = {"id": "health-one", "kind": "instance.inspect", "instanceId": "same", "payload": {}} + errors = [] + def request(method, path, body=None): + if method == "GET": + return {"operations": [health_op if "lane=health" in path else file_op]} + if path.endswith("file-one") and body["state"] == "completed": + receipt_waiting.set() + release_receipt.wait(3) + return {} + def dispatch(operation): + if operation["id"] == "health-one": + health_done.set() + self.agent.stopping.set() + return {"ok": True} + def run(lane): + try: self.agent.read_loop(lane) + except Exception as error: errors.append(error) + self.agent.request, self.agent.dispatch = request, dispatch + files = threading.Thread(target=run, args=("read",)) + health = threading.Thread(target=run, args=("health",)) + files.start() + try: + self.assertTrue(receipt_waiting.wait(2)) + health.start() + self.assertTrue(health_done.wait(2), "Health waited for a file receipt") + finally: + self.agent.stopping.set() + release_receipt.set() + files.join(3) + if health.ident is not None: health.join(3) + self.assertFalse(files.is_alive()) + self.assertFalse(health.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(self.agent.journal.get("health-one")["state"], "completed") + def test_instance_mutations_and_global_mutations_exclude_observations(self): operation = {"instanceId": "one"} entered = threading.Event() diff --git a/docs/maintenance.md b/docs/maintenance.md index 74ecfa0..b901ec7 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -126,6 +126,11 @@ Drift includes missing promoted files, changed bytes, and extra files in managed script, config, and plugin directories. Config comparison includes the composed baseline, overlays, and the selected map. Unmanaged default config is exempt until a baseline is promoted. Missing mounts are scan errors, not empty directories. +Routine drift scans run every five minutes per server, reducing file traffic +while health observations continue every minute. **Check now**, a replacement +registration, or a new lifecycle submission bypasses the drift interval. Failed +file scans retry on the next observer pass; post-rebuild verification still reads +fresh files. Drift repair creates a Campaign capturing observed differences and requests a Provisioning rebuild. The Campaign observes that rebuild without requesting diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 1cae003..0067f3a 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -5,6 +5,13 @@ instance never uses Komodo or a filesystem mounted into the controller. The controller owns desired state, campaigns, collections, safety queues and history; the agent owns local execution and its durable operation journal. +Fleet health checks query RCON even when container inspection is unavailable. A +fresh successful RCON reply can establish game health while container state stays +unknown; confirmed stopped, unhealthy, and restarting states still take precedence. +Agent 1.4.4 retries transient failures of the read-only `status` query up to three +times, with 250 ms and 500 ms delays. Commands that change the game are not retried. +Status replies still expire normally, and native timeout diagnostics are retained. + ## Install and launch 1. Run the controller with persistent `/data` storage. Set `EC_PUBLIC_URL` to its @@ -479,9 +486,14 @@ of which task delayed a previous deployment. Campaign, beta and maintenance observers run independently of provisioning. New requests and completed agent commands wake provisioning; an arrival during a -pass schedules another pass. File drift scans use bounded batches of four reads -per scan to reduce agent polling round trips. Startup and disruption safety gates -remain enforced at command admission. +pass schedules another pass. Routine file drift scans run every five minutes; +manual checks bypass that interval. Overlapping identical native observations +share one operation until it completes, without caching results for later checks. +Agent 1.4.5 reserves two observation workers for instance health and read-only RCON, +and one for files, logs, and host inventory. Background reads arrive in batches of +up to four. File receipt uploads release the instance lock before contacting the +controller. Older agents retain the shared read lane until upgraded. Startup and +disruption safety gates remain enforced at command admission. Older completed operations whose progress snapshots were already discarded cannot recover missing check results from this change. Their surviving events diff --git a/src/lib/server-observation.ts b/src/lib/server-observation.ts index fe67756..f8bfa5f 100644 --- a/src/lib/server-observation.ts +++ b/src/lib/server-observation.ts @@ -43,7 +43,8 @@ export function selectServerObservation(server: T, if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry; const raw = server.rcon_status ?? server; const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null }; - if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running') || (!server.container_state && server.health === 'offline')) { + const containerUnknown = server.container_state === 'unknown'; + if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) { return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null }; } const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at; @@ -69,7 +70,9 @@ export function selectServerObservation(server: T, } const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after); const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error; - const containerExpired = !!server.container_checked_at && !containerFresh; + // A current authenticated RCON reply can establish game liveness when + // inspection is unavailable, without claiming the container was inspected. + const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh; const age = now - Date.parse(raw.last_seen_at ?? ''); return { ...result, health: fresh && !containerExpired ? 'online' : after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' }; diff --git a/src/lib/server/backends/native.ts b/src/lib/server/backends/native.ts index 4e4e6f4..a6816d2 100644 --- a/src/lib/server/backends/native.ts +++ b/src/lib/server/backends/native.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js'; import { trackActivity } from '../pipeline/activity.js'; +import { latestGameSubmission } from '../fleet/lifecycle-evidence.js'; export interface NativeInstance { id: string; server: string; host_id: string; generation: number; library_id: string; @@ -17,10 +18,27 @@ export function nativeDeployment(id: string): boolean { return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id); } -/** Only the agent receives host paths, Docker access, or game credentials. */ +const observations = new Map>(); +const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'logs.read']); + +/** Share overlapping observations only, never completed results or mutations. */ export async function nativeRequest>(instance: NativeInstance, kind: string, payload: Record = {}, key?: string): Promise { const host = getHost(instance.host_id); if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.'); + const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command))); + if (key || !readOnly) return executeRequest(instance, kind, payload, key); + // A probe after a lifecycle submission must not join an old-process read. + const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server, + latestGameSubmission(instance.server), kind, payload]); + const pending = observations.get(identity); + if (pending) return pending as Promise; + const request = executeRequest(instance, kind, payload).finally(() => { observations.delete(identity); }); + observations.set(identity, request); + return request; +} + +/** Only the agent receives host paths, Docker access, or game credentials. */ +async function executeRequest(instance: NativeInstance, kind: string, payload: Record, key?: string): Promise { const operation = queueHostOperation(instance.host_id, kind, payload, { instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID() }); diff --git a/src/lib/server/fleet/health.ts b/src/lib/server/fleet/health.ts index 4633181..9a1dba9 100644 --- a/src/lib/server/fleet/health.ts +++ b/src/lib/server/fleet/health.ts @@ -11,7 +11,7 @@ const STALE_MAX_MS = 120_000; */ export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health { if (containerState === 'starting' || containerState === 'restarting') return 'starting'; - if (containerState && containerState !== 'running') return 'offline'; + if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline'; if (!lastSeenAt) return 'offline'; const age = Date.now() - new Date(lastSeenAt).getTime(); if (freshStatusTimestamp(lastSeenAt, Date.now())) { diff --git a/src/lib/server/fleet/status.ts b/src/lib/server/fleet/status.ts index 9650304..50fa4fe 100644 --- a/src/lib/server/fleet/status.ts +++ b/src/lib/server/fleet/status.ts @@ -49,7 +49,7 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) { last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster }; const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name); const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state; - const ready = !provisioning && (!state || state === 'running'); + const ready = !provisioning && (!state || state === 'running' || state === 'unknown'); const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state); return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health, status_source: null as 'telemetry' | 'rcon' | null, diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index d390139..2bc4e69 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -348,7 +348,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record queueMicrotask(() => workEvents.emit(hostId)); return operation; } -export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] { +export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] { const host = requireHost(hostId); if (host.mode === 'disabled' || !host.connected) return []; // Older agents hold activity.lock across the entire delivered batch, including @@ -362,7 +362,13 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr // consults its durable journal before acknowledging or executing a replay. const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read') OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`; - const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`; + const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`; + const healthLane = host.inventory.capabilities?.includes('operations.health-lane'); + const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}` + : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`; + // Health never queues behind a delivered file batch. Keep background reads + // in small batches to amortize delivery without handing out a long backlog. + const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25; return database().transaction(() => { const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') ${laneFilter} @@ -370,7 +376,7 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr AND (kind!='instance.cancel-create' OR ?) AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id) - ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`) + ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`) .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[]; const deliveredAt = now(); for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); diff --git a/src/lib/server/integrations/rcon.ts b/src/lib/server/integrations/rcon.ts index 515d6a3..755ae70 100644 --- a/src/lib/server/integrations/rcon.ts +++ b/src/lib/server/integrations/rcon.ts @@ -382,7 +382,19 @@ export async function rconStatusResult(server: string): Promise => { try { - const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' }); + const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' }); + if (reply.ok === false) { + const failures: Record = { + unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' }, + unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' }, + timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' }, + authentication: { reason: 'bad-reply', detail: 'wrong RCON password' }, + 'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' } + }; + return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? { + reason: 'bad-reply' as const, detail: 'Agent RCON query failed.' + }) }; + } const players = parsePlayerCount(reply.text); if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) }; const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text); diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 3d02c8f..629eb51 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -16,6 +16,7 @@ import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident import { maintenanceAllowed, setMaintenancePolicy } from './policies.js'; export const WATCHDOG_SCAN_MS = 60_000; +export const WATCHDOG_DRIFT_SCAN_MS = 5 * 60_000; export const WATCHDOG_FAILURE_MS = 120_000; export const WATCHDOG_RECOVERY_MS = 300_000; const iso = () => new Date().toISOString(); @@ -240,7 +241,7 @@ async function observe(server: string, operationId?: string): Promise { +async function scanServer(server: string, forceScan: boolean): Promise { const row = getServer(server); if (!row || row.desired_state === 'stopped') return; if (scanSuppressed(server) || protectedWork(server)) return; @@ -265,8 +266,19 @@ async function scanServer(server: string): Promise { } // Unknown container health is an observation failure, not proof that a rebuild is safe. if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return; + // File trees change far less often than liveness. Persist the last successful + // scan per registration/process so health checks do not rescan every minute. + // Manual checks and new lifecycle submissions always get fresh file evidence. + const driftKey = `watchdog_drift_scan:${server}`; + const instance = nativeInstance(server); + const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]); + let previous: { identity?: string; at?: string } = {}; + try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ } + const age = Date.now() - Date.parse(previous?.at ?? ''); + if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return; const prepared = await prepareDriftCampaign(server); if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return; + setSetting(driftKey, JSON.stringify({ identity, at: iso() })); if (!prepared.plan.files.length) { const current = active(server, 'drift'); if (current) transition(current.id, 'resolved', 'Server files match the promoted setup'); @@ -305,7 +317,7 @@ async function run(forceScan: boolean): Promise { async function pump() { while (cursor < servers.length) { const server = servers[cursor++]; - try { await scanServer(server.name); } + try { await scanServer(server.name, forceScan); } catch (err) { if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`); } diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 33a69b7..b125ace 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -150,10 +150,14 @@ async function refreshServerStatus(server: string): Promise { if (unchanged()) { updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown'; updates.container_checked_at = new Date().toISOString(); - if (!container.ok) { + // An unavailable inspection is not evidence that the game stopped. + // Probe it just as a manual RCON check would; explicit lifecycle or + // unhealthy states still take precedence over game replies. + const unavailable = updates.container_state === 'unknown'; + if (!container.ok && !unavailable) { updates.last_error = null; updates.last_reply_snippet = null; - } else if (!telemetryEvidence(srv.name)) { + } else if (unavailable || !telemetryEvidence(srv.name)) { flush(); const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined })); diff --git a/src/routes/api/agent/operations/+server.ts b/src/routes/api/agent/operations/+server.ts index 729e649..735442d 100644 --- a/src/routes/api/agent/operations/+server.ts +++ b/src/routes/api/agent/operations/+server.ts @@ -6,7 +6,7 @@ export async function GET({ request }: { request: Request }) { const host = authenticateAgent(request); reconcileHostPreparation(host.id); const params = new URL(request.url).searchParams; - const lane = params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all'; + const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all'; let operations = pendingHostOperations(host.id, lane); if (!operations.length && params.get('wait') === '20') { await waitForHostWork(host.id, request.signal); diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 872eddb..3079a19 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.3" +VERSION = "1.4.5" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -1092,7 +1092,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1126,11 +1126,20 @@ class Agent: bind = saved.get("bindAddress", "0.0.0.0") host = "127.0.0.1" if bind in ("0.0.0.0", "") else "::1" if bind == "::" else bind family = socket.AF_INET6 if ":" in host else socket.AF_INET + # Retry only the read-only occupancy query. A missing UDP reply must + # never cause commands with effects to be replayed. + for attempt in range(3 if command == "status" else 1): + result = self.rcon_attempt(family, host, int(saved["port"]), password, command) + if result["ok"] or result.get("reason") not in ("timeout", "unreachable") or attempt == 2 or command != "status": + return result + time.sleep(0.25 * 2 ** attempt) + + def rcon_attempt(self, family, host, port, password, command): chunks = [] with socket.socket(family, socket.SOCK_DGRAM) as sock: sock.settimeout(3) try: - sock.connect((host, int(saved["port"]))) + sock.connect((host, port)) sock.send(b"\xff\xff\xff\xffrcon " + password.encode() + b" " + command.encode() + b"\n") while len(chunks) < 64 and sum(map(len, chunks)) < 1024 * 1024: chunks.append(sock.recv(65535)) @@ -2016,7 +2025,11 @@ class Agent: key = operation.get("instanceId") return self.instance_locks.setdefault(key, threading.RLock()) if key else contextlib.nullcontext() - def read_loop(self): + @staticmethod + def health_operation(operation): + return operation.get("kind") == "instance.inspect" or (operation.get("kind") == "rcon.execute" and operation.get("payload", {}).get("command") in ("status", "getstatus", "version")) + + def read_loop(self, lane="read"): # Each lane owns its SQLite connection and never writes executing_id, # checkpoints, or current_operation used by the mutation state machine. journal = Journal(self.root / "agent" / "operations.sqlite3", recover=False) @@ -2026,10 +2039,13 @@ class Agent: if self.update_pending(): self.stopping.wait(3) continue - response = self.request("GET", "/api/agent/operations?lane=read&wait=20") + response = self.request("GET", f"/api/agent/operations?lane={lane}&wait=20") + handled = False for operation in response.get("operations", []): if self.stopping.is_set() or not self.read_operation(operation): continue + if lane == "health" and not self.health_operation(operation): + continue with self.read_claim_lock: if operation["id"] in self.read_claims: continue @@ -2038,19 +2054,26 @@ class Agent: with self.activity(read_only=True) as acquired: if not acquired or self.update_pending(): continue - with self.read_gate.read(), self.instance_lock(operation): - row, _ = journal.accept(operation) - if row["state"] in ("accepted", "unknown", "running"): - self.report(journal.update(operation["id"], "running")) + row, _ = journal.accept(operation) + if row["state"] in ("accepted", "unknown", "running"): + self.report(journal.update(operation["id"], "running")) + with self.read_gate.read(), self.instance_lock(operation): try: result = self.dispatch(operation) row = journal.update(operation["id"], "completed", result) except Exception as err: row = journal.update(operation["id"], "failed", error=str(err) if isinstance(err, AgentError) else "Observation unavailable") - self.report(row) + # Receipt delivery can take seconds; it must not + # hold the instance lock needed by health checks. + self.report(row) + handled = True finally: with self.read_claim_lock: self.read_claims.discard(operation["id"]) + if response.get("operations") and not handled: + # A legacy controller may return another lane's work or + # a read already claimed locally. Avoid a busy poll loop. + self.stopping.wait(0.25) except (AgentError, OSError, ValueError, KeyError, sqlite3.Error): self.stopping.wait(3) finally: @@ -2109,7 +2132,8 @@ class Agent: self.stopping.wait(3) heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True) heartbeats.start() - readers = [threading.Thread(target=self.read_loop, daemon=True) for _ in range(2)] + # Preserve two concurrent health checks even while files are being read. + readers = [threading.Thread(target=self.read_loop, args=(lane,), daemon=True) for lane in ("health", "health", "read")] for reader in readers: reader.start() while not self.stopping.is_set(): diff --git a/static/agent/manifest.json b/static/agent/manifest.json index cd2248e..06bff47 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.4.3", + "version": "1.4.5", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "9ca9bff1a42ab838b175329c6a8458e18053429efe2f050f477a9ad9c598410f", + "sha256": "bd4afa2ec3ab6a3022dba99d0580b4b6b1ba5622ae4138035e3e3862885a97bd", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts index 3c38856..bf07be2 100644 --- a/tests/unit/native-hosts.test.ts +++ b/tests/unit/native-hosts.test.ts @@ -15,6 +15,24 @@ async function enrolled() { } describe('native host enrollment and scoped authentication', () => { + it('reserves health delivery while a file backlog is pending and bounds background batches', async () => { + const api = await enrolled(); + api.heartbeat(api.host.id, { ...inventory, capabilities: ['operations.health-lane'] }); + const files = Array.from({ length: 30 }, (_, i) => api.queueHostOperation(api.host.id, 'files.read', { path: `file-${i}` })); + const status = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'status' }); + const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); + const mutation = api.queueHostOperation(api.host.id, 'rcon.execute', { command: 'map_restart' }); + expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([status.id]); + api.reportHostOperation(api.host.id, status.id, { state: 'completed', result: {} }); + expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]); + expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual(files.slice(0, 4).map(op => op.id)); + expect(api.pendingHostOperations(api.host.id, 'write').map(op => op.id)).toEqual([mutation.id]); + // Rolling upgrades: older agents still receive health through read. + api.heartbeat(api.host.id, inventory); + files.forEach(file => api.reportHostOperation(api.host.id, file.id, { state: 'completed', result: {} })); + expect(api.pendingHostOperations(api.host.id, 'read').map(op => op.id)).toEqual([inspect.id]); + }); + it('separates observation and mutation delivery and wakes long polls on newly queued work', async () => { const api = await enrolled(); const inspect = api.queueHostOperation(api.host.id, 'host.inspect', {}); diff --git a/tests/unit/native-observations.test.ts b/tests/unit/native-observations.test.ts new file mode 100644 index 0000000..05c4f30 --- /dev/null +++ b/tests/unit/native-observations.test.ts @@ -0,0 +1,76 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { await fixture.cleanup(); }); + +async function setup() { + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name: 'Observation host' }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("INSERT INTO servers(name,komodo_name) VALUES('test','')").run(); + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('instance','test',?,'library','image',28960,?)`).run(host.id, new Date().toISOString()); + const native = await import('../../src/lib/server/backends/native.js'); + const instance = native.nativeInstance('test')!; + return { ...hosts, ...native, db: db(), host, instance }; +} + +describe('overlapping native observations', () => { + it.each(['files.read', 'files.list', 'files.stat', 'logs.read', 'instance.inspect'])('shares %s only until completion', async kind => { + const api = await setup(); + const requests = Array.from({ length: 4 }, () => api.nativeRequest(api.instance, kind, { path: 'same' })); + const operations = api.listHostOperations(api.host.id); + expect(operations).toHaveLength(1); + api.reportHostOperation(api.host.id, operations[0].id, { state: 'completed', result: { value: 'first' } }); + expect(await Promise.all(requests)).toEqual(Array(4).fill({ value: 'first' })); + const fresh = api.nativeRequest(api.instance, kind, { path: 'same' }); + const next = api.listHostOperations(api.host.id)[0]; + expect(next.id).not.toBe(operations[0].id); + api.reportHostOperation(api.host.id, next.id, { state: 'completed', result: { value: 'changed' } }); + expect(await fresh).toEqual({ value: 'changed' }); + }); + + it('does not merge different paths, mutations, or reads across lifecycle submissions', async () => { + const api = await setup(); + const requests = [api.nativeRequest(api.instance, 'files.read', { path: 'one' }), + api.nativeRequest(api.instance, 'files.read', { path: 'two' }), + api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' }), + api.nativeRequest(api.instance, 'rcon.execute', { command: 'map_restart' })]; + api.db.prepare("INSERT INTO restart_rollout(server,requested_at) VALUES('test',?)").run(new Date().toISOString()); + requests.push(api.nativeRequest(api.instance, 'files.read', { path: 'one' })); + const operations = api.listHostOperations(api.host.id); + expect(operations).toHaveLength(5); + for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, { state: 'completed', result: {} }); + await Promise.all(requests); + }); + + it('releases failed observations so the next request can recover', async () => { + const api = await setup(); + const first = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); + const rejected = expect(first).rejects.toThrow('unavailable'); + api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'failed', error: 'unavailable' }); + await rejected; + const next = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); + expect(api.listHostOperations(api.host.id)).toHaveLength(2); + api.reportHostOperation(api.host.id, api.listHostOperations(api.host.id)[0].id, { state: 'completed', result: {} }); + await next; + }); + + it('keeps replacement generations separate and rejects the old result', async () => { + const api = await setup(); + const old = api.nativeRequest(api.instance, 'files.read', { path: 'one' }); + const rejected = expect(old).rejects.toThrow('Instance was replaced'); + api.db.prepare("UPDATE native_instances SET generation=2 WHERE id='instance'").run(); + const current = api.nativeRequest(api.nativeInstance('test')!, 'files.read', { path: 'one' }); + const operations = api.listHostOperations(api.host.id); + expect(operations).toHaveLength(2); + for (const operation of operations) api.reportHostOperation(api.host.id, operation.id, { + state: 'completed', result: { generation: operation.generation } + }); + await rejected; + expect(await current).toEqual({ generation: 2 }); + }); +}); diff --git a/tests/unit/rcon-status.test.ts b/tests/unit/rcon-status.test.ts index 8733c2b..37bf216 100644 --- a/tests/unit/rcon-status.test.ts +++ b/tests/unit/rcon-status.test.ts @@ -5,6 +5,7 @@ import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js'; import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js'; import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js'; +import { nativeInstance, nativeRequest } from '../../src/lib/server/backends/native.js'; vi.mock('node:dgram', () => ({ createSocket: vi.fn() })); vi.mock('../../src/lib/server/config.js', () => ({ @@ -14,6 +15,7 @@ vi.mock('../../src/lib/server/config.js', () => ({ vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() })); vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() })); vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() })); +vi.mock('../../src/lib/server/backends/native.js', () => ({ nativeInstance: vi.fn(), nativeRequest: vi.fn() })); class FakeSocket extends EventEmitter { send = vi.fn(); @@ -47,6 +49,28 @@ afterEach(() => { }); describe('RCON status transport', () => { + it.each([ + ['timeout', 'timeout', 'Agent received no reply'], + ['unreachable', 'send-error', 'Agent could not reach'], + ['unconfigured', 'no-config', 'Agent RCON password'], + ['authentication', 'bad-reply', 'wrong RCON password'], + ['invalid-reply', 'bad-reply', 'Agent RCON reply is missing'] + ])('preserves the native agent %s diagnosis', async (agentReason, reason, detail) => { + server.native_instance_id = 'instance'; + vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never); + vi.mocked(nativeRequest).mockResolvedValue({ ok: false, reason: agentReason, text: '' }); + expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason, detail: expect.stringContaining(detail) }); + expect(nativeRequest).toHaveBeenCalledOnce(); + expect(sockets).toHaveLength(0); + }); + + it('uses the successful native reply after agent-side recovery', async () => { + server.native_instance_id = 'instance'; + vi.mocked(nativeInstance).mockReturnValue({ id: 'instance', generation: 1, host_id: 'host', port: 28960 } as never); + vi.mocked(nativeRequest).mockResolvedValue({ ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' }); + expect(await rconStatusResult(server.name)).toMatchObject({ ok: true, status: { players: 0, map: 'mp_kino' } }); + }); + it('shares overlapping health and occupancy probes and probes again after completion', async () => { const health = rconStatusResult(server.name); const occupancy = rconGetPlayerCount(server.name); diff --git a/tests/unit/server-observation.test.ts b/tests/unit/server-observation.test.ts index a4803a8..837b691 100644 --- a/tests/unit/server-observation.test.ts +++ b/tests/unit/server-observation.test.ts @@ -30,13 +30,22 @@ describe('shared server observation', () => { expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) }); }); - it.each(['starting', 'restarting', 'unhealthy', 'exited', 'unknown'])('telemetry cannot override container state %s', state => { + it.each(['starting', 'restarting', 'unhealthy', 'exited'])('telemetry cannot override container state %s', state => { const selected = selectServerObservation(server({ health: state === 'starting' || state === 'restarting' ? 'starting' : 'offline', container_state: state }), report(), now); expect(selected.health).not.toBe('online'); expect(selected.last_players).toBeNull(); expect(selected.roster).toBeNull(); }); + it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => { + const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) }); + expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' }); + expect(selectServerObservation(raw, report(46_000), now + 46_000).health).toBe('stale'); + expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline'); + expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline'); + expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline'); + }); + it('requires fresh container evidence and reports after the latest lifecycle submission', () => { expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online'); expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning'); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts index 0374f8f..a10fa58 100644 --- a/tests/unit/server-status.test.ts +++ b/tests/unit/server-status.test.ts @@ -379,7 +379,7 @@ describe('database fleet and container readiness', () => { expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null }); }); - it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => { + it.each(['unhealthy', 'exited', 'stopped', 'missing'])('does not poll RCON with container state %s', async (state) => { const { inspect, rcon, fanoutStatus } = await setup(); inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); @@ -389,6 +389,30 @@ describe('database fleet and container readiness', () => { expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null }); }); + it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => { + const { inspect, rcon, fanoutStatus, getServer } = await setup(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out')); + else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' }); + await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + const events: unknown[] = []; + const unsubscribe = subscribe(event => events.push(event)); + try { await fanoutStatus(); } finally { unsubscribe(); } + expect(rcon).toHaveBeenCalledOnce(); + expect(getServer('managed')?.container_state).toBe('unknown'); + expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 2, status_source: 'rcon' }); + expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ + health: 'online', players: 2, containerState: 'unknown' + })] })); + + rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + await fanoutStatus(); + expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' }); + }); + it('waits for an EC restart request to settle before trusting the old healthy container', async () => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index 580af17..f188efa 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -68,6 +68,41 @@ async function sustainedFailure() { } describe('Watchdog detection and ownership', () => { + it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => { + const drift = await import('../../src/lib/server/fleet/drift.js'); + const scan = vi.spyOn(drift, 'prepareDriftCampaign'); + await watchdog.watchdogTick(); + expect(scan).toHaveBeenCalledTimes(1); + for (let minute = 1; minute < 5; minute++) { + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(); + } + expect(scan).toHaveBeenCalledTimes(1); + expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(); + expect(scan).toHaveBeenCalledTimes(2); + await watchdog.watchdogTick(true); + expect(scan).toHaveBeenCalledTimes(3); + advance(watchdog.WATCHDOG_SCAN_MS); + database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server); + await watchdog.watchdogTick(); + expect(scan).toHaveBeenCalledTimes(4); + scan.mockRestore(); + }); + + it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => { + const drift = await import('../../src/lib/server/fleet/drift.js'); + const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable')); + await watchdog.watchdogTick(); + expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable'); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(); + expect(scan).toHaveBeenCalledTimes(2); + expect(watchdog.watchdogOverview().scanError).toBeNull(); + scan.mockRestore(); + }); + it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => { expect(watchdog.watchdogOverview().enabled).toBe(true); down();