josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve verified health when provisioning completes

Browse Source
This commit is contained in:
josh
2026-10-06 15:48:38 -04:00
parent 6999a33aef
commit 57d2bd11db
9 changed files with 256 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/monitoring-snapshot.ts
+8 -6
View File
@@ -8,6 +8,7 @@ import type { ServerRow } from './servers.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { serverObservation } from './status.js';
import { recoverySettling } from '../provisioning/restarts.js';
const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
@@ -21,21 +22,21 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
(SELECT MAX(at) FROM (SELECT MAX(d.submitted_at) AS at FROM server_deployments d
WHERE d.server=s.name AND d.server_deleted_at IS NULL AND d.action IN ('create','start','restart','rebuild')
UNION ALL SELECT r.requested_at)) AS submission,
r.requested_at AS restart_at,r.healthy_at AS restart_healthy,t.received_at AS telemetry_at,t.payload AS telemetry_json
r.requested_at AS restart_at,r.healthy_at AS restart_healthy,r.operation_id AS restart_operation_id,t.received_at AS telemetry_at,t.payload AS telemetry_json
FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL
LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
interface Row extends ServerRow {
instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
restart_at: string | null; restart_healthy: string | null; telemetry_at: string | null; telemetry_json: string | null;
restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
}
interface Record {
server: ServerRow; native: NativeInstance | null;
host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
restartAt: string | null; restartHealthy: string | null;
restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
}
/** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
@@ -48,7 +49,7 @@ export class MonitoringSnapshot {
if (server === undefined) this.records.clear(); else this.records.delete(server);
for (const row of rows) {
const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
restart_at,restart_healthy,telemetry_at,telemetry_json,...fields } = row;
restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
let telemetry: GameTelemetry | null = null;
if (telemetry_json) try {
@@ -60,7 +61,7 @@ export class MonitoringSnapshot {
host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
restartAt: restart_at, restartHealthy: restart_healthy });
restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
}
}
get(server: string) { return this.records.get(server); }
@@ -71,7 +72,8 @@ export class MonitoringSnapshot {
}
settling(server: string) {
const record = this.get(server);
return !!record?.restartAt && !record.restartHealthy && Date.now() - Date.parse(record.restartAt) < 15_000;
return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
}
telemetryEvidence(server: string) {
const record = this.get(server), report = record?.telemetry;
src/lib/server/fleet/status-evidence.ts
+7 -3
View File
@@ -1,7 +1,7 @@
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js';
import { latestGameSubmission } from './lifecycle-evidence.js';
export { latestGameSubmission } from './lifecycle-evidence.js';
@@ -59,9 +59,13 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
return { players: null, source: null, observedAt: new Date().toISOString() };
}
export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & (
{ source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
| { source: 'telemetry'; receivedAt: string });
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
VerifiedGameStatus | { ok: false; reason: string; observedAt: string }
> {
const current = telemetryEvidence(server, after);
if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
@@ -70,7 +74,7 @@ export async function probeServerStatus(server: string, after?: string | null):
const rcon = await rconStatusResult(server);
const latest = telemetryEvidence(server, after);
if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() };
if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() };
reason = rcon.reason;
} catch (err) { reason = (err as Error).message; }
const telemetry = telemetryEvidence(server, after);
src/lib/server/fleet/status-updates.ts
+22
View File
@@ -0,0 +1,22 @@
import type { RconStatusResult } from '../integrations/rcon.js';
import type { ServerObservation } from '../../server-observation.js';
import type { ServerRow } from './servers.js';
/** Monitoring and provisioning publish the same complete RCON observation. */
export function successfulStatusFields(result: Extract<RconStatusResult, { ok: true }>, observedAt: string): Partial<ServerRow> {
return {
last_checked_at: observedAt, last_seen_at: observedAt,
last_players: result.status.players, last_map: result.status.map,
last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster),
last_reply_snippet: result.rawReply ?? null, last_error: null, rcon_failures: 0
};
}
/** Occupancy timers and match clearing follow the same source as the UI. */
export function activityFields(current: ServerRow, status: ServerObservation): Partial<ServerRow> {
if (status.health !== 'online' || status.last_players === null) return {};
return status.last_players > 0
? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null }
: { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at,
game_started_at: null, game_ended_at: null };
}
src/lib/server/fleet/status.ts
+6 -6
View File
@@ -1,7 +1,7 @@
import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
import { classifyHealth } from './health.js';
import { isProvisioning } from './provisioning.js';
import { restartSettling } from '../provisioning/restarts.js';
import { restartSettling, recoverySettling } from '../provisioning/restarts.js';
import { type ServerRow } from './servers.js';
import { telemetryReport, latestGameSubmission } from './status-evidence.js';
import { latestGameSubmissions } from './lifecycle-evidence.js';
@@ -13,11 +13,11 @@ interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTe
/** One set of evidence reads per snapshot, rather than four queries per server. */
export function observationReader() {
const handle = db();
const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments
WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
{ server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
{ id: string; server: string; action: string; state: string }[]).map(row => [row.server, row]));
const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as
{ server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row]));
const submissions = latestGameSubmissions();
const reports = new Map<string, GameTelemetry>();
for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
@@ -32,7 +32,7 @@ export function observationReader() {
const operation = operations.get(server.name), restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
settling: recoverySettling(restart, operation),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
});
};