diff --git a/src/lib/server/fleet/monitoring-snapshot.ts b/src/lib/server/fleet/monitoring-snapshot.ts index 371dcb3..8865259 100644 --- a/src/lib/server/fleet/monitoring-snapshot.ts +++ b/src/lib/server/fleet/monitoring-snapshot.ts @@ -8,6 +8,7 @@ import type { ServerRow } from './servers.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { serverObservation } from './status.js'; +import { recoverySettling } from '../provisioning/restarts.js'; const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at']; const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id, @@ -21,21 +22,21 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in (SELECT MAX(at) FROM (SELECT MAX(d.submitted_at) AS at FROM server_deployments d WHERE d.server=s.name AND d.server_deleted_at IS NULL AND d.action IN ('create','start','restart','rebuild') UNION ALL SELECT r.requested_at)) AS submission, - r.requested_at AS restart_at,r.healthy_at AS restart_healthy,t.received_at AS telemetry_at,t.payload AS telemetry_json + r.requested_at AS restart_at,r.healthy_at AS restart_healthy,r.operation_id AS restart_operation_id,t.received_at AS telemetry_at,t.payload AS telemetry_json FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`; interface Row extends ServerRow { instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number; lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null; - restart_at: string | null; restart_healthy: string | null; telemetry_at: string | null; telemetry_json: string | null; + restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null; } interface Record { server: ServerRow; native: NativeInstance | null; host: Pick | null; lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null; lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null; - restartAt: string | null; restartHealthy: string | null; + restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null; } /** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */ @@ -48,7 +49,7 @@ export class MonitoringSnapshot { if (server === undefined) this.records.clear(); else this.records.delete(server); for (const row of rows) { const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission, - restart_at,restart_healthy,telemetry_at,telemetry_json,...fields } = row; + restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row; const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS; let telemetry: GameTelemetry | null = null; if (telemetry_json) try { @@ -60,7 +61,7 @@ export class MonitoringSnapshot { host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline', inventory: parseHostInventory(row.host_id!, host_inventory) } : null, lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry, - restartAt: restart_at, restartHealthy: restart_healthy }); + restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id }); } } get(server: string) { return this.records.get(server); } @@ -71,7 +72,8 @@ export class MonitoringSnapshot { } settling(server: string) { const record = this.get(server); - return !!record?.restartAt && !record.restartHealthy && Date.now() - Date.parse(record.restartAt) < 15_000; + return recoverySettling(record?.restartAt ? { requested_at: record.restartAt, + healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle); } telemetryEvidence(server: string) { const record = this.get(server), report = record?.telemetry; diff --git a/src/lib/server/fleet/status-evidence.ts b/src/lib/server/fleet/status-evidence.ts index 380dd03..ce7c48c 100644 --- a/src/lib/server/fleet/status-evidence.ts +++ b/src/lib/server/fleet/status-evidence.ts @@ -1,7 +1,7 @@ import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; -import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; +import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js'; import { latestGameSubmission } from './lifecycle-evidence.js'; export { latestGameSubmission } from './lifecycle-evidence.js'; @@ -59,9 +59,13 @@ export async function probePlayerEvidence(server: string): Promise } + | { source: 'telemetry'; receivedAt: string }); + /** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */ export async function probeServerStatus(server: string, after?: string | null): Promise< - { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string } + VerifiedGameStatus | { ok: false; reason: string; observedAt: string } > { const current = telemetryEvidence(server, after); if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() }; @@ -70,7 +74,7 @@ export async function probeServerStatus(server: string, after?: string | null): const rcon = await rconStatusResult(server); const latest = telemetryEvidence(server, after); if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() }; - if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() }; + if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() }; reason = rcon.reason; } catch (err) { reason = (err as Error).message; } const telemetry = telemetryEvidence(server, after); diff --git a/src/lib/server/fleet/status-updates.ts b/src/lib/server/fleet/status-updates.ts new file mode 100644 index 0000000..3bf8404 --- /dev/null +++ b/src/lib/server/fleet/status-updates.ts @@ -0,0 +1,22 @@ +import type { RconStatusResult } from '../integrations/rcon.js'; +import type { ServerObservation } from '../../server-observation.js'; +import type { ServerRow } from './servers.js'; + +/** Monitoring and provisioning publish the same complete RCON observation. */ +export function successfulStatusFields(result: Extract, observedAt: string): Partial { + return { + last_checked_at: observedAt, last_seen_at: observedAt, + last_players: result.status.players, last_map: result.status.map, + last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster), + last_reply_snippet: result.rawReply ?? null, last_error: null, rcon_failures: 0 + }; +} + +/** Occupancy timers and match clearing follow the same source as the UI. */ +export function activityFields(current: ServerRow, status: ServerObservation): Partial { + if (status.health !== 'online' || status.last_players === null) return {}; + return status.last_players > 0 + ? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null } + : { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at, + game_started_at: null, game_ended_at: null }; +} diff --git a/src/lib/server/fleet/status.ts b/src/lib/server/fleet/status.ts index c492e94..d3e33f7 100644 --- a/src/lib/server/fleet/status.ts +++ b/src/lib/server/fleet/status.ts @@ -1,7 +1,7 @@ import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js'; import { classifyHealth } from './health.js'; import { isProvisioning } from './provisioning.js'; -import { restartSettling } from '../provisioning/restarts.js'; +import { restartSettling, recoverySettling } from '../provisioning/restarts.js'; import { type ServerRow } from './servers.js'; import { telemetryReport, latestGameSubmission } from './status-evidence.js'; import { latestGameSubmissions } from './lifecycle-evidence.js'; @@ -13,11 +13,11 @@ interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTe /** One set of evidence reads per snapshot, rather than four queries per server. */ export function observationReader() { const handle = db(); - const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments + const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as - { server: string; action: string; state: string }[]).map(row => [row.server, row])); - const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as - { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row])); + { id: string; server: string; action: string; state: string }[]).map(row => [row.server, row])); + const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as + { server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row])); const submissions = latestGameSubmissions(); const reports = new Map(); for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as @@ -32,7 +32,7 @@ export function observationReader() { const operation = operations.get(server.name), restart = restarts.get(server.name); return serverObservation(server, { provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state), - settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000), + settling: recoverySettling(restart, operation), telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null }); }; diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 3bf30c7..4f01bcb 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -9,6 +9,7 @@ import type { RconFailure } from '../integrations/rcon.js'; import { setServerFields, type ServerRow } from '../fleet/servers.js'; import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, monitoringIdentity, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js'; import { serverObservation } from '../fleet/status.js'; +import { successfulStatusFields, activityFields } from '../fleet/status-updates.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js'; import { recordMetrics } from '../fleet/metrics.js'; import { selectedZombieMap } from '../../map-aliases.js'; @@ -220,13 +221,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, updates.last_reply_snippet = result.rawReply ?? null; if (result.ok) { capacity = result.status.capacity ?? null; - updates.last_players = result.status.players; - updates.last_map = result.status.map; - updates.last_gametype = result.status.gametype; - updates.last_roster = JSON.stringify(result.status.roster); - updates.last_seen_at = now; - updates.last_error = null; - updates.rcon_failures = 0; + Object.assign(updates, successfulStatusFields(result, now)); } else { // Only distinct accepted checks in the current lifecycle count. // Skipped/superseded checks return above without changing the streak. @@ -247,18 +242,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, } const current = { ...(snapshot.get(srv.name)?.server ?? srv), ...updates } as ServerRow; const status = snapshot.observe(current); - // Occupancy timers and match clearing follow the same source as the UI. - if (status.health === 'online' && status.last_players !== null) { - if (status.last_players > 0) { - updates.active_since = current.active_since ?? status.last_seen_at; - updates.inactive_since = null; - } else { - updates.active_since = null; - updates.inactive_since = current.inactive_since ?? status.last_seen_at; - updates.game_started_at = null; - updates.game_ended_at = null; - } - } + Object.assign(updates, activityFields(current, status)); flush(); return capacity; } diff --git a/src/lib/server/provisioning/health-handoff.ts b/src/lib/server/provisioning/health-handoff.ts new file mode 100644 index 0000000..1064ac7 --- /dev/null +++ b/src/lib/server/provisioning/health-handoff.ts @@ -0,0 +1,37 @@ +import { ONLINE_MAX_MS } from '../../server-freshness.js'; +import { monitoringIdentity } from '../fleet/health-observations.js'; +import { getServer, setServerFields, type ServerRow } from '../fleet/servers.js'; +import { latestGameSubmission, telemetryReport, type VerifiedGameStatus } from '../fleet/status-evidence.js'; +import { serverObservation } from '../fleet/status.js'; +import { activityFields, successfulStatusFields } from '../fleet/status-updates.js'; + +/** Called inside the completion transaction; evidence must still describe this process. */ +export function handoffVerifiedHealth(server: string, identity: string, + container: { state: string; observedAt: string }, game: VerifiedGameStatus): boolean { + const current = getServer(server); + if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) return false; + const fresh = (at: string) => { + const age = Date.now() - Date.parse(at); + return age >= 0 && age <= ONLINE_MAX_MS; + }; + const gameAt = game.source === 'telemetry' ? game.receivedAt : game.observedAt; + const submission = latestGameSubmission(server); + if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) return false; + + const updates: Partial = {}; + if (!current.container_checked_at || current.container_checked_at <= container.observedAt) { + updates.container_state = container.state; + updates.container_checked_at = container.observedAt; + } + if (game.source === 'rcon' && (!current.last_checked_at || current.last_checked_at <= game.observedAt)) { + Object.assign(updates, successfulStatusFields(game.rcon, game.observedAt)); + } + const candidate = { ...current, ...updates }; + // Evaluate the health readers will see after the operation and recovery slot + // complete. Newer contradictory evidence must keep verification pending. + const status = serverObservation(candidate, { provisioning: false, settling: false, + submission, telemetry: telemetryReport(server) }); + if (status.health !== 'online') return false; + setServerFields(server, { ...updates, ...activityFields(candidate, status) }); + return true; +} diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 768e2eb..13e8552 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -23,6 +23,8 @@ import { trackActivity } from '../pipeline/activity.js'; import { fleetSettings } from '../settings.js'; import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js'; +import { monitoringIdentity } from '../fleet/health-observations.js'; +import { handoffVerifiedHealth } from './health-handoff.js'; import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js'; import { recordScript, removeScript } from '../fleet/inventory.js'; import { sha256Buffer } from '../fleet/files.js'; @@ -544,7 +546,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) }); }; deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks }); + const verificationIdentity = monitoringIdentity(job.server); const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect'); + const containerObservedAt = new Date().toISOString(); if (!stillActive(job)) return; if (job.action === 'stop') { const stopped = ['stopped','exited','created','dead'].includes(health.state); @@ -584,9 +588,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; } if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; } } - db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server); - recordProgress(job.id, { blocker: null, checks }); - complete(job); return; + const completed = db().transaction(() => { + if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity, + { state: health.state, observedAt: containerObservedAt }, status)) return false; + recordProgress(job.id, { blocker: null, checks }); + complete(job); + return true; + })(); + if (!completed) hold(job, 'Waiting for current health evidence before completing verification.'); + return; } const reason = eligibility(job); if (reason) { hold(job, reason); return; } diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index 7ec03cf..ef6d487 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -23,11 +23,24 @@ export function restartRecoveryPending(server: string): boolean { return slotFor(server)?.healthy_at === null; } -/** The queued restart may still expose the previous healthy container. */ +type SettlingSlot = Pick; +interface SettlingOperation { id: string; action: string; state: string } + +/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */ +export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean { + if (!slot || slot.healthy_at) return false; + if (slot.operation_id) return operation?.id === slot.operation_id + && ['start', 'restart', 'rebuild'].includes(operation.action) + && ['running', 'awaiting-health'].includes(operation.state); + // Legacy recovery slots have no operation whose verification can be followed. + return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS; +} + export function restartSettling(server: string): boolean { const slot = slotFor(server); - return !!(slot && !slot.healthy_at - && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS); + const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments + WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined; + return recoverySettling(slot, operation); } export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null { diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index f4fab99..d09d11e 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -30,7 +30,7 @@ beforeEach(async () => { return { healthy: remote.state === 'running', state: remote.state, exists: true } as never; }); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); +afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.unstubAllEnvs(); }); async function prepareLibrary(game: 't5' | 't6' = 't5') { const libs = await import('../../src/lib/server/game-files/index.js'); @@ -74,6 +74,149 @@ async function detail(id: string) { return getDeployment(id)!.deployment; } + +async function observations(server: string) { + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + const { serverObservation, observationReader } = await import('../../src/lib/server/fleet/status.js'); + const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js'); + const row = getServer(server)!; + return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)]; +} + +describe('provisioning health handoff', () => { + it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => { + const job = await queue(); await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + let id = job.deploymentId; + if (action !== 'create') { + await acknowledge(id); + db().prepare('UPDATE servers SET container_state=? WHERE name=?').run(action === 'start' ? 'stopped' : 'running', job.name); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const request = requestLifecycle(job.name, action); + if (!request.ok) throw new Error(request.error); + id = request.deploymentId; + await advance(); + } + const old = new Date(Date.now() - 120_000).toISOString(); + db().prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]', + last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`).run(old, old, old, job.name); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(Date.now() + 1000); + const observedAt = new Date().toISOString(); + const roster = [{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }]; + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: 'verified status', + status: { players: 1, map: remote.map, gametype: 'zom', roster } }); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); + const visible: string[] = []; + const unsubscribe = subscribe(event => { + if (event.type === 'fleet-invalidated' && getDeployment(id)?.deployment.state === 'done') { + visible.push(serverObservation(getServer(job.name)!).health); + } + }); + try { await acknowledge(id); } finally { unsubscribe(); } + expect((await detail(id)).state).toBe('done'); + expect(visible).toEqual(['online']); + expect(getServer(job.name)).toMatchObject({ last_checked_at: observedAt, last_seen_at: observedAt, + last_players: 1, last_roster: JSON.stringify(roster), last_reply_snippet: 'verified status', + last_error: null, rcon_failures: 0, active_since: observedAt, inactive_since: null }); + for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 1, roster }); + }); + + it('completes directly from persisted telemetry without inventing an RCON observation', async () => { + const job = await queue(); await advance(); + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } }); + vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); + const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); + const { gameEvent } = await import('../helpers/telemetry.js'); + const receivedAt = new Date().toISOString(); + saveTelemetry({ ...gameEvent('heartbeat', 0, 0, job.name), receivedAt }); + await advance(); + expect((await detail(job.deploymentId)).state).toBe('done'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)).toMatchObject({ last_seen_at: null, inactive_since: receivedAt }); + for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', status_source: 'telemetry', last_seen_at: receivedAt }); + }); + + it('rechecks expired verification evidence instead of renewing its timestamps', async () => { + const job = await queue(); await advance(); + vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + vi.mocked(probeVersion).mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 91_000); + heartbeat(hostId, inventory()); + return 5354; + }); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('awaiting-health'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)?.last_seen_at).toBeNull(); + for (const status of await observations(job.name)) expect(status.health).toBe('provisioning'); + await advance(); + expect((await detail(job.deploymentId)).state).toBe('done'); + expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString()); + }); + + it.each(['instance', 'lifecycle'] as const)('discards verification when the %s changes during later checks', async change => { + const job = await queue(); await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + vi.mocked(probeVersion).mockImplementationOnce(async () => { + if (change === 'instance') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); + else db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() + 1000).toISOString(), job.deploymentId); + return 5354; + }); + await acknowledge(job.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('awaiting-health'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)).toMatchObject({ last_seen_at: null, container_checked_at: null }); + }); + + it.each(['online', 'offline'] as const)('preserves newer %s monitoring evidence received during version verification', async health => { + const job = await queue(); await advance(); + vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + let newer: string; + vi.mocked(probeVersion).mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + 1000); newer = new Date().toISOString(); + db().prepare(`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?, + last_players=2,last_roster='[]',last_error=NULL WHERE name=?`).run(health === 'online' ? 'running' : 'stopped', newer, newer, newer, job.name); + return 5354; + }); + await acknowledge(job.deploymentId); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + expect(getServer(job.name)).toMatchObject({ last_seen_at: newer!, container_checked_at: newer!, last_players: 2 }); + expect((await detail(job.deploymentId)).state).toBe(health === 'online' ? 'done' : 'awaiting-health'); + if (health === 'online') for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 2 }); + }); + + it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const request = requestLifecycle(job.name, 'restart'); + if (!request.ok) throw new Error(request.error); + await advance(); + vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 20_000); + for (const status of await observations(job.name)) expect(status.health).toBe('starting'); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const state of ['failed', 'cancelled', 'interrupted']) { + db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, request.deploymentId); + for (const status of await observations(job.name)) expect(status.health).toBe('offline'); + } + db().prepare("UPDATE server_deployments SET state='running' WHERE id=?").run(request.deploymentId); + await acknowledge(request.deploymentId); + expect((await detail(request.deploymentId)).state).toBe('done'); + for (const status of await observations(job.name)) expect(status.health).toBe('online'); + }); +}); + describe('native instance provisioning', () => { it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => { const job = await queue();