josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native health polling and isolate file operation traffic

Browse Source
This commit is contained in:
josh
2026-10-04 21:50:29 -04:00
parent 8a31de92fa
commit 81f088739c
22 changed files with 446 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server-observation.ts
+5 -2
View File
@@ -43,7 +43,8 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
if (server.telemetry && (!report || Date.parse(server.telemetry.receivedAt) > Date.parse(report.receivedAt))) report = server.telemetry;
const raw = server.rcon_status ?? server;
const result: T = { ...server, ...raw, telemetry: report, status_source: raw.last_seen_at ? 'rcon' : null };
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running') || (!server.container_state && server.health === 'offline')) {
const containerUnknown = server.container_state === 'unknown';
if (['provisioning','starting'].includes(server.health) || (server.container_state && server.container_state !== 'running' && !containerUnknown) || (!server.container_state && server.health === 'offline')) {
return { ...result, health: server.health, last_error: null, last_players: null, roster: null, status_source: null };
}
const checked = server.container_checked_at ?? server.last_checked_at ?? raw.last_seen_at;
@@ -69,7 +70,9 @@ export function selectServerObservation<T extends ServerObservation>(server: T,
}
const after = !server.telemetry_after || Date.parse(raw.last_seen_at ?? '') > Date.parse(server.telemetry_after);
const fresh = after && freshStatusTimestamp(raw.last_seen_at, now) && !raw.last_error;
const containerExpired = !!server.container_checked_at && !containerFresh;
// A current authenticated RCON reply can establish game liveness when
// inspection is unavailable, without claiming the container was inspected.
const containerExpired = !containerUnknown && !!server.container_checked_at && !containerFresh;
const age = now - Date.parse(raw.last_seen_at ?? '');
return { ...result, health: fresh && !containerExpired ? 'online'
: after && age >= 0 && age <= 120_000 ? 'stale' : 'offline' };
src/lib/server/backends/native.ts
+19 -1
View File
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js';
import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
export interface NativeInstance {
id: string; server: string; host_id: string; generation: number; library_id: string;
@@ -17,10 +18,27 @@ export function nativeDeployment(id: string): boolean {
return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id);
}
/** Only the agent receives host paths, Docker access, or game credentials. */
const observations = new Map<string, Promise<unknown>>();
const readKinds = new Set(['instance.inspect', 'files.read', 'files.stat', 'files.list', 'logs.read']);
/** Share overlapping observations only, never completed results or mutations. */
export async function nativeRequest<T = Record<string, unknown>>(instance: NativeInstance, kind: string, payload: Record<string, unknown> = {}, key?: string): Promise<T> {
const host = getHost(instance.host_id);
if (!host || host.connectionState !== 'online') throw new Error('Host agent is offline; instance state is unknown.');
const readOnly = readKinds.has(kind) || (kind === 'rcon.execute' && ['status', 'getstatus', 'version'].includes(String(payload.command)));
if (key || !readOnly) return executeRequest<T>(instance, kind, payload, key);
// A probe after a lifecycle submission must not join an old-process read.
const identity = JSON.stringify([instance.host_id, instance.id, instance.generation, instance.server,
latestGameSubmission(instance.server), kind, payload]);
const pending = observations.get(identity);
if (pending) return pending as Promise<T>;
const request = executeRequest<T>(instance, kind, payload).finally(() => { observations.delete(identity); });
observations.set(identity, request);
return request;
}
/** Only the agent receives host paths, Docker access, or game credentials. */
async function executeRequest<T>(instance: NativeInstance, kind: string, payload: Record<string, unknown>, key?: string): Promise<T> {
const operation = queueHostOperation(instance.host_id, kind, payload, {
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
});
src/lib/server/fleet/health.ts
+1 -1
View File
@@ -11,7 +11,7 @@ const STALE_MAX_MS = 120_000;
*/
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running') return 'offline';
if (containerState && containerState !== 'running' && containerState !== 'unknown') return 'offline';
if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime();
if (freshStatusTimestamp(lastSeenAt, Date.now())) {
src/lib/server/fleet/status.ts
+1 -1
View File
@@ -49,7 +49,7 @@ export function serverObservation(server: ServerRow, evidence?: Evidence) {
last_gametype: server.last_gametype, last_seen_at: server.last_seen_at, last_error: server.last_error, roster };
const provisioning = evidence ? evidence.provisioning : isProvisioning(server.name);
const state = (evidence ? evidence.settling : restartSettling(server.name)) ? 'restarting' : server.container_state;
const ready = !provisioning && (!state || state === 'running');
const ready = !provisioning && (!state || state === 'running' || state === 'unknown');
const health = provisioning ? 'provisioning' : classifyHealth(server.last_seen_at, server.last_error, state);
return selectServerObservation({ ...server, ...rcon, rcon_status: rcon, container_state: state, health,
status_source: null as 'telemetry' | 'rcon' | null,
src/lib/server/hosts/index.ts
+9 -3
View File
@@ -348,7 +348,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
queueMicrotask(() => workEvents.emit(hostId));
return operation;
}
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'write' = 'all'): HostOperation[] {
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
@@ -362,7 +362,13 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
const laneFilter = lane === 'all' ? '' : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}`;
const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
// Health never queues behind a delivered file batch. Keep background reads
// in small batches to amortize delivery without handing out a long backlog.
const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
return database().transaction(() => {
const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
@@ -370,7 +376,7 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'wr
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[];
const deliveredAt = now();
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
src/lib/server/integrations/rcon.ts
+13 -1
View File
@@ -382,7 +382,19 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
const endpoint = { host: `agent:${native.host_id}`, port: native.port };
const request = (async (): Promise<RconStatusResult> => {
try {
const reply = await nativeRequest<{ text: string }>(native, 'rcon.execute', { command: 'status' });
const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' });
if (reply.ok === false) {
const failures: Record<string, { reason: RconFailure; detail: string }> = {
unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' },
unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' },
timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' },
authentication: { reason: 'bad-reply', detail: 'wrong RCON password' },
'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' }
};
return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? {
reason: 'bad-reply' as const, detail: 'Agent RCON query failed.'
}) };
}
const players = parsePlayerCount(reply.text);
if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
src/lib/server/maintenance/recovery.ts
+14 -2
View File
@@ -16,6 +16,7 @@ import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
export const WATCHDOG_SCAN_MS = 60_000;
export const WATCHDOG_DRIFT_SCAN_MS = 5 * 60_000;
export const WATCHDOG_FAILURE_MS = 120_000;
export const WATCHDOG_RECOVERY_MS = 300_000;
const iso = () => new Date().toISOString();
@@ -240,7 +241,7 @@ async function observe(server: string, operationId?: string): Promise<Observatio
return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
}
async function scanServer(server: string): Promise<void> {
async function scanServer(server: string, forceScan: boolean): Promise<void> {
const row = getServer(server);
if (!row || row.desired_state === 'stopped') return;
if (scanSuppressed(server) || protectedWork(server)) return;
@@ -265,8 +266,19 @@ async function scanServer(server: string): Promise<void> {
}
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
// File trees change far less often than liveness. Persist the last successful
// scan per registration/process so health checks do not rescan every minute.
// Manual checks and new lifecycle submissions always get fresh file evidence.
const driftKey = `watchdog_drift_scan:${server}`;
const instance = nativeInstance(server);
const identity = JSON.stringify([row.created_at, instance?.id, instance?.generation, latestGameSubmission(server)]);
let previous: { identity?: string; at?: string } = {};
try { previous = JSON.parse(setting(driftKey) ?? '{}'); } catch { /* rescan legacy/invalid state */ }
const age = Date.now() - Date.parse(previous?.at ?? '');
if (!forceScan && previous?.identity === identity && age >= 0 && age < WATCHDOG_DRIFT_SCAN_MS) return;
const prepared = await prepareDriftCampaign(server);
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
setSetting(driftKey, JSON.stringify({ identity, at: iso() }));
if (!prepared.plan.files.length) {
const current = active(server, 'drift');
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
@@ -305,7 +317,7 @@ async function run(forceScan: boolean): Promise<void> {
async function pump() {
while (cursor < servers.length) {
const server = servers[cursor++];
try { await scanServer(server.name); }
try { await scanServer(server.name, forceScan); }
catch (err) {
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
}
src/lib/server/pipeline/worker.ts
+6 -2
View File
@@ -150,10 +150,14 @@ async function refreshServerStatus(server: string): Promise<number | null> {
if (unchanged()) {
updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
updates.container_checked_at = new Date().toISOString();
if (!container.ok) {
// An unavailable inspection is not evidence that the game stopped.
// Probe it just as a manual RCON check would; explicit lifecycle or
// unhealthy states still take precedence over game replies.
const unavailable = updates.container_state === 'unknown';
if (!container.ok && !unavailable) {
updates.last_error = null;
updates.last_reply_snippet = null;
} else if (!telemetryEvidence(srv.name)) {
} else if (unavailable || !telemetryEvidence(srv.name)) {
flush();
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
src/routes/api/agent/operations/+server.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export async function GET({ request }: { request: Request }) {
const host = authenticateAgent(request);
reconcileHostPreparation(host.id);
const params = new URL(request.url).searchParams;
const lane = params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
let operations = pendingHostOperations(host.id, lane);
if (!operations.length && params.get('wait') === '20') {
await waitForHostWork(host.id, request.signal);