Scope server snapshots and keep completed history off fleet refreshes
This commit is contained in:
@@ -2,19 +2,31 @@ import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
||||
import { db } from '../db/index.js';
|
||||
import type { FleetOperation } from '../../fleet-overview.js';
|
||||
|
||||
/** Read persisted work, including recent completions for one-shot lobby watches. */
|
||||
/** Fleet clients need only active work and unresolved failures. */
|
||||
export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
return readOperations(now);
|
||||
}
|
||||
|
||||
/** Recent completions are loaded only by the server's own page. */
|
||||
export function serverOperations(server: string, now = Date.now()): FleetOperation[] {
|
||||
return readOperations(now, server);
|
||||
}
|
||||
|
||||
function readOperations(now: number, server?: string): FleetOperation[] {
|
||||
const since = new Date(now - 86_400_000).toISOString();
|
||||
const bindings = { server: server ?? null, since };
|
||||
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
|
||||
const handle = db();
|
||||
const operations: FleetOperation[] = [];
|
||||
const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
|
||||
.all() as { server: string; requested_at: string; error: string | null }[];
|
||||
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
|
||||
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
|
||||
const recovering = new Map(slots.map(slot => [slot.server, slot]));
|
||||
const representedRecovery = new Set<string>();
|
||||
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
|
||||
ds.restart_state, ds.error, ds.restarted_at, ds.players
|
||||
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
|
||||
WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
|
||||
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
|
||||
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
|
||||
id: string; action: string; started_at: string; server: string; restart_state: string;
|
||||
error: string | null; restarted_at: string | null; players: number | null;
|
||||
}[];
|
||||
@@ -34,19 +46,19 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
href: `/campaigns/${encodeURIComponent(row.id)}` });
|
||||
}
|
||||
// Only the newest lifecycle operation can still require action on a server.
|
||||
// Separate branches let SQLite use state and finish indexes; UNION deduplicates
|
||||
// recent warnings that are also the newest operation. Unary + keeps terminal
|
||||
// branches on finished_at and rowid instead of searching historical states.
|
||||
// Separate branches use state/rowid indexes for fleet reads and the server
|
||||
// index for detail history. Completed work is never loaded for the full fleet.
|
||||
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
||||
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
||||
${server === undefined ? '' : `UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND server = @server
|
||||
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
|
||||
UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ?
|
||||
UNION
|
||||
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning')
|
||||
AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`)
|
||||
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
|
||||
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
|
||||
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
|
||||
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
|
||||
error: string | null; started_at: string }[];
|
||||
for (const row of lifecycle) {
|
||||
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
|
||||
@@ -59,8 +71,8 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
|
||||
}
|
||||
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
|
||||
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
|
||||
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
|
||||
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
|
||||
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
|
||||
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
|
||||
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
|
||||
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
|
||||
@@ -71,9 +83,9 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
|
||||
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
|
||||
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
|
||||
}
|
||||
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
|
||||
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
|
||||
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
|
||||
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
|
||||
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
|
||||
for (const job of maintenance) {
|
||||
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
|
||||
server: job.server, state: 'attention', startedAt: job.requested_at,
|
||||
|
||||
@@ -1,26 +1,32 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { listServers } from './servers.js';
|
||||
import { observationReader } from './status.js';
|
||||
import { getServer, listServers } from './servers.js';
|
||||
import { observationReader, serverObservation } from './status.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { runningVersions } from '../integrations/plutonium.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import { serverOperations } from './overview.js';
|
||||
|
||||
export function fleetSnapshot() {
|
||||
export function fleetSnapshot(serverName?: string) {
|
||||
const bindings = serverName === undefined ? [] : [serverName];
|
||||
const rconHost = config.rcon.host;
|
||||
const hosts = listHosts();
|
||||
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
|
||||
const versions = new Map(runningVersions().map(version => [version.server, version]));
|
||||
const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
|
||||
const versions = new Map(runningVersions(serverName).map(version => [version.server, version]));
|
||||
const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
|
||||
.all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance]));
|
||||
const holds = new Map<string, { target: string; betaId: number }[]>();
|
||||
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
|
||||
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
|
||||
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
|
||||
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')
|
||||
${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as
|
||||
{ server: string; target: string; betaId: number }[]) {
|
||||
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
|
||||
}
|
||||
const observe = observationReader();
|
||||
const servers = listServers().map((s) => {
|
||||
const observe = serverName === undefined ? observationReader() : serverObservation;
|
||||
const selected = serverName === undefined ? undefined : getServer(serverName);
|
||||
const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
|
||||
const servers = rows.map((s) => {
|
||||
const instance = s.native_instance_id ? instances.get(s.name) : null;
|
||||
return {
|
||||
...observe(s),
|
||||
@@ -36,7 +42,7 @@ export function fleetSnapshot() {
|
||||
return {
|
||||
servers,
|
||||
serverNow: Date.now(),
|
||||
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
|
||||
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
|
||||
integrations: {
|
||||
native: hosts.length > 0 || !komodoConfigured(),
|
||||
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
|
||||
@@ -49,3 +55,6 @@ export function fleetSnapshot() {
|
||||
};
|
||||
}
|
||||
|
||||
export function serverSnapshot(name: string) {
|
||||
return { ...fleetSnapshot(name), operations: serverOperations(name) };
|
||||
}
|
||||
|
||||
@@ -20,13 +20,18 @@ export function freshRelease(): number | null {
|
||||
const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN;
|
||||
return !latest.error && age >= 0 && age < HOUR ? latest.revision : null;
|
||||
}
|
||||
export function runningVersions(): PlutoniumVersion[] {
|
||||
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
|
||||
export function runningVersions(serverName?: string): PlutoniumVersion[] {
|
||||
const bindings = serverName === undefined ? [] : [serverName];
|
||||
const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`)
|
||||
.all(...bindings) as PlutoniumVersion[];
|
||||
const target = freshRelease();
|
||||
const byServer = new Map(versions.map(version => [version.server, version]));
|
||||
const epochs = latestGameSubmissions();
|
||||
const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
|
||||
return listServers().map(server => {
|
||||
const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]);
|
||||
const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
|
||||
.all(...bindings) as { server: string }[]).map(row => row.server));
|
||||
const selected = serverName === undefined ? undefined : getServer(serverName);
|
||||
const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
|
||||
return servers.map(server => {
|
||||
const value = byServer.get(server.name)
|
||||
?? { server: server.name, revision: null, checked_at: null, error: null };
|
||||
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
|
||||
|
||||
+18
-10
@@ -6,12 +6,14 @@ import { browser } from '$app/environment';
|
||||
import { statusByServer, statusReceivedAt, syncServerTime, serverTimeNow, matchEventsByServer, telemetryByServer, uiNow } from './servers.js';
|
||||
import { type FleetServer, type FleetOperation } from '$lib/fleet-overview.js';
|
||||
|
||||
export const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
// Each server detail view owns a scoped snapshot; it must not replace the shared fleet.
|
||||
export function createFleetStore(serverName?: string) {
|
||||
const fleet = writable<{ servers: FleetServer[]; operations: FleetOperation[]; receivedAt: number }>({
|
||||
servers: [], operations: [], receivedAt: 0
|
||||
});
|
||||
export const fleetError = writable('');
|
||||
export const fleetRefreshing = writable(false);
|
||||
export const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
const fleetError = writable('');
|
||||
const fleetRefreshing = writable(false);
|
||||
const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
|
||||
|
||||
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
|
||||
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
|
||||
@@ -23,7 +25,7 @@ const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow]
|
||||
});
|
||||
return times.map(time => `${freshStatusTimestamp(time ?? null, now)}:${now - Date.parse(time ?? '') >= 0 && now - Date.parse(time ?? '') <= 120_000}`).join(',');
|
||||
});
|
||||
export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const fleetServers = derived([fleet, statusByServer, statusReceivedAt, matchEventsByServer, telemetryByServer, freshnessPhase], ([snapshot, live, received, matches, reports]) => {
|
||||
const now = get(uiNow);
|
||||
const next = snapshot.servers.map(server => {
|
||||
const candidate = live.get(server.name);
|
||||
@@ -63,7 +65,7 @@ export const fleetServers = derived([fleet, statusByServer, statusReceivedAt, ma
|
||||
|
||||
|
||||
let snapshotServerNow = 0;
|
||||
export function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
|
||||
if (!browser || snapshot.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = snapshot.serverNow;
|
||||
syncServerTime(snapshot.serverNow);
|
||||
@@ -76,7 +78,7 @@ function getIntegrations() { return get(fleetIntegrations); }
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let refreshRequested = false;
|
||||
export function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
if (!browser) return Promise.resolve();
|
||||
if (inFlight) { refreshRequested ||= afterCurrent; return inFlight; }
|
||||
inFlight = (async () => {
|
||||
@@ -87,15 +89,17 @@ export function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
try {
|
||||
const results = await Promise.allSettled([
|
||||
(async () => {
|
||||
const response = await fetch('/api/servers', { signal: controller.signal });
|
||||
const response = await fetch(serverName === undefined ? '/api/servers' : `/api/servers/${encodeURIComponent(serverName)}`, { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh servers. Retrying automatically.');
|
||||
const servers = await response.json();
|
||||
if (servers.serverNow < snapshotServerNow) return;
|
||||
snapshotServerNow = servers.serverNow;
|
||||
syncServerTime(servers.serverNow); fleetIntegrations.set(servers.integrations);
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt }));
|
||||
fleet.update(value => ({ ...value, servers: servers.servers, receivedAt: requestedAt,
|
||||
operations: serverName === undefined ? value.operations : servers.operations }));
|
||||
})(),
|
||||
(async () => {
|
||||
if (serverName !== undefined) return;
|
||||
const response = await fetch('/api/fleet', { signal: controller.signal });
|
||||
if (!response.ok) throw new Error('Could not refresh operations. Retrying automatically.');
|
||||
const operations = await response.json();
|
||||
@@ -120,7 +124,7 @@ export function refreshFleet(afterCurrent = false): Promise<void> {
|
||||
}
|
||||
|
||||
let stop: (() => void) | null = null;
|
||||
export function startFleetMonitor(): () => void {
|
||||
function startFleetMonitor(): () => void {
|
||||
if (!browser || stop) return () => {};
|
||||
const stopPolling = pollVisible(() => refreshFleet(), 60_000, !get(fleet).receivedAt || Date.now() - get(fleet).receivedAt > 30_000);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
@@ -159,3 +163,7 @@ export function startFleetMonitor(): () => void {
|
||||
};
|
||||
return stop;
|
||||
}
|
||||
return { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor };
|
||||
}
|
||||
|
||||
export const { fleet, fleetServers, fleetError, fleetRefreshing, fleetIntegrations, hydrateFleet, refreshFleet, startFleetMonitor } = createFleetStore();
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
$effect(() => {
|
||||
if (!sessionReady) return;
|
||||
const path = $page.url.pathname;
|
||||
if (!(path === '/' || /^\/(servers|scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
|
||||
if (!(path === '/' || /^\/(scripts|configs|plugins)\//.test(path) || path === '/betas/new')) return;
|
||||
if ('fleetSnapshot' in $page.data && $page.data.fleetSnapshot) hydrateFleet($page.data.fleetSnapshot);
|
||||
return startFleetMonitor();
|
||||
});
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { kick } from '$lib/server/pipeline/worker.js';
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { destroyServer } from '$lib/server/provisioning/index.js';
|
||||
import { serverSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
|
||||
export function GET({ params }: { params: { name: string } }) {
|
||||
return json(serverSnapshot(params.name));
|
||||
}
|
||||
|
||||
export async function DELETE({ params }: { params: { name: string } }) {
|
||||
const name = params.name;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { fleetSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
import { fleetOperations } from '$lib/server/fleet/overview.js';
|
||||
import { serverSnapshot } from '$lib/server/fleet/snapshot.js';
|
||||
|
||||
export function load() { return { fleetSnapshot: { ...fleetSnapshot(), operations: fleetOperations() } }; }
|
||||
export function load({ params }: { params: { name: string } }) {
|
||||
return { serverSnapshot: serverSnapshot(params.name) };
|
||||
}
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
<script lang="ts">
|
||||
let { data }: { data: import('./$types').PageData } = $props();
|
||||
const snapshot = $derived($fleet.receivedAt ? $fleet : data.fleetSnapshot);
|
||||
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.fleetSnapshot.servers);
|
||||
import RuntimeVersion from '$ui/components/RuntimeVersion.svelte';
|
||||
import MatchHistory from '$ui/components/MatchHistory.svelte';
|
||||
import RoundTelemetry from '$ui/components/RoundTelemetry.svelte';
|
||||
import TelemetryDiagnostics from '$ui/components/TelemetryDiagnostics.svelte';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import { fetchCampaigns } from '$ui/campaigns.js';
|
||||
import { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } from '$ui/stores/fleet.js';
|
||||
import { createFleetStore } from '$ui/stores/fleet.js';
|
||||
import { freshServer, fleetIdentity } from '$lib/fleet-overview.js';
|
||||
import { serverTitle } from '$ui/server-labels.js';
|
||||
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
|
||||
@@ -109,9 +107,17 @@
|
||||
}
|
||||
|
||||
let name = $derived($page.params.name ?? '');
|
||||
const detail = $derived(createFleetStore(name));
|
||||
const { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } = $derived(detail);
|
||||
$effect(() => {
|
||||
detail.hydrateFleet(data.serverSnapshot);
|
||||
return detail.startFleetMonitor();
|
||||
});
|
||||
const snapshot = $derived($fleet.receivedAt ? $fleet : data.serverSnapshot);
|
||||
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.serverSnapshot.servers);
|
||||
let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null);
|
||||
const native = $derived(!!server?.native_instance_id);
|
||||
let integrations = $derived($fleetIntegrations ?? data.fleetSnapshot.integrations);
|
||||
let integrations = $derived($fleetIntegrations ?? data.serverSnapshot.integrations);
|
||||
let history = $state<DeploymentRow[]>([]);
|
||||
let inventory = $state<InventoryEntry[]>([]);
|
||||
let komodoProbe = $state<KomodoProbeResult | null>(null);
|
||||
@@ -610,7 +616,7 @@
|
||||
</div>
|
||||
{:else if $fleetError}
|
||||
<div class="ui-error" role="alert">{$fleetError}<Button onclick={() => refreshFleet(true)}>Retry</Button></div>
|
||||
{:else if $fleet.receivedAt || data.fleetSnapshot}
|
||||
{:else if $fleet.receivedAt || data.serverSnapshot}
|
||||
<EmptyState title="Server not found" description="This server may have been removed from the fleet." />
|
||||
{:else}
|
||||
<div class="loading">Loading…</div>
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { get } from 'svelte/store';
|
||||
import type { FleetOperation } from '../../src/lib/fleet-overview.js';
|
||||
|
||||
vi.mock('$app/environment', () => ({ browser: true }));
|
||||
|
||||
@@ -33,6 +35,44 @@ async function setup() {
|
||||
}
|
||||
|
||||
describe('fleet event refreshes', () => {
|
||||
it('refreshes server history through one scoped request without replacing the fleet snapshot', async () => {
|
||||
const store = await import('../../src/lib/ui/stores/fleet.js');
|
||||
const integrations = { rcon: true, komodo: true, rconTimeoutMs: 2000, defaultRconHost: '' };
|
||||
const snapshot = { servers: [], operations: [], integrations, serverNow: Date.now() };
|
||||
store.hydrateFleet(snapshot);
|
||||
const originalFleet = get(store.fleet);
|
||||
const detail = store.createFleetStore('one / two');
|
||||
detail.hydrateFleet(snapshot);
|
||||
stop = detail.startFleetMonitor();
|
||||
const history: FleetOperation[] = [{ key: 'deployment:done', group: 'deployment:done', kind: 'deployment',
|
||||
server: 'one / two', state: 'done', label: 'Complete', detail: 'Complete', href: '/provisioning/done', startedAt: new Date().toISOString() }];
|
||||
vi.mocked(fetch).mockResolvedValue(Response.json({ ...snapshot, operations: history }));
|
||||
changed();
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fetch).toHaveBeenCalledWith('/api/servers/one%20%2F%20two', expect.anything());
|
||||
expect(get(detail.fleet).operations).toEqual(history);
|
||||
expect(get(store.fleet)).toBe(originalFleet);
|
||||
expect(get(store.fleet).operations).toEqual([]);
|
||||
});
|
||||
|
||||
it('isolates late responses when navigating between server detail stores', async () => {
|
||||
const { createFleetStore, fleet } = await import('../../src/lib/ui/stores/fleet.js');
|
||||
const first = createFleetStore('first');
|
||||
const second = createFleetStore('second');
|
||||
let finish!: (response: Response) => void;
|
||||
vi.mocked(fetch).mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
|
||||
const oldRequest = first.refreshFleet();
|
||||
const newSnapshot = { serverNow: Date.now(), servers: [], integrations: {}, operations: [] };
|
||||
vi.mocked(fetch).mockResolvedValue(Response.json(newSnapshot));
|
||||
await second.refreshFleet();
|
||||
const current = get(second.fleet);
|
||||
finish(Response.json(newSnapshot)); await oldRequest;
|
||||
expect(get(second.fleet)).toBe(current);
|
||||
expect(get(fleet).receivedAt).toBe(0);
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('coalesces bursts and refreshes at most once every two seconds with a trailing refresh', async () => {
|
||||
await setup();
|
||||
changed(); changed(); changed();
|
||||
|
||||
@@ -64,8 +64,10 @@ describe('persisted Fleet operations', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at,detail) VALUES ('server','update','failed',5354,?,'Revision not verified')").run(iso());
|
||||
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now)).toEqual([expect.objectContaining({ kind: 'maintenance', state: 'attention', detail: 'Revision not verified' })]);
|
||||
expect(serverOperations('server', now)).toEqual(fleetOperations(now));
|
||||
expect(serverOperations('unrelated', now)).toEqual([]);
|
||||
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at) VALUES ('server','update','queued',5354,?)").run(iso(1000));
|
||||
expect(fleetOperations(now)).toEqual([]);
|
||||
});
|
||||
@@ -76,7 +78,7 @@ describe('persisted Fleet operations', () => {
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('one', 'promote', ?)").run(iso(-2 * 86_400_000));
|
||||
handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players) VALUES ('one', 'server', 1, 'pending', 4)").run();
|
||||
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now)).toEqual([expect.objectContaining({ label: 'Waiting for an empty lobby', state: 'active', href: '/campaigns/one' })]);
|
||||
handle.prepare("UPDATE deployment_servers SET restarted_at = ?, players = NULL").run(iso());
|
||||
handle.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('server', ?)").run(iso());
|
||||
@@ -86,18 +88,20 @@ describe('persisted Fleet operations', () => {
|
||||
expect(fleetOperations(now)[0].label).toBe('Recovering');
|
||||
db().prepare("UPDATE deployment_servers SET restart_state = 'done', checked_at = ?").run(iso());
|
||||
db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(iso());
|
||||
expect(fleetOperations(now)[0].state).toBe('done');
|
||||
expect(fleetOperations(now)).toEqual([]);
|
||||
expect(serverOperations('server', now)[0].state).toBe('done');
|
||||
});
|
||||
|
||||
it('keeps lifecycle failures actionable until superseded and retains successful completions', async () => {
|
||||
it('keeps lifecycle failures actionable until superseded and retains completions only for that server', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
const insert = handle.prepare("INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, triggered_by) VALUES (?, ?, ?, ?, ?, ?, ?, 'test')");
|
||||
insert.run('failed', 'server', 'create', 'failed', 'Creating container', iso(-86_400_001), iso(-86_400_001));
|
||||
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now)[0].state).toBe('attention');
|
||||
insert.run('retry', 'server', 'create', 'done', 'Ready', iso(), iso());
|
||||
expect(fleetOperations(now)).toEqual([expect.objectContaining({ key: 'deployment:retry', state: 'done' })]);
|
||||
expect(fleetOperations(now)).toEqual([]);
|
||||
expect(serverOperations('server', now)).toEqual([expect.objectContaining({ key: 'deployment:retry', state: 'done' })]);
|
||||
insert.run('queued', 'new-server', 'create', 'queued', 'Waiting to launch', iso(), null);
|
||||
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ server: 'new-server', label: 'Waiting to launch', state: 'active' }));
|
||||
});
|
||||
@@ -107,7 +111,7 @@ describe('persisted Fleet operations', () => {
|
||||
const handle = db();
|
||||
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
|
||||
handle.prepare("INSERT INTO restart_rollout (server, requested_at, error) VALUES ('server', ?, 'RCON not answering')").run(iso());
|
||||
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now)[0]).toMatchObject({ kind: 'recovery', state: 'attention', detail: 'RCON not answering' });
|
||||
handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run();
|
||||
handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run();
|
||||
@@ -115,9 +119,11 @@ describe('persisted Fleet operations', () => {
|
||||
handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'awaiting-match')").run();
|
||||
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'beta', label: 'Observing a match', href: '/betas/1' }));
|
||||
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'recovery', state: 'attention' }));
|
||||
expect(serverOperations('server', now)).toEqual(fleetOperations(now));
|
||||
expect(serverOperations('unrelated', now)).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves lifecycle selection across cutoff, deletion, supersession, and overlapping warning branches', async () => {
|
||||
it('keeps only active work and unresolved failures across deletion and supersession', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const handle = db();
|
||||
const since = iso(-86_400_000);
|
||||
@@ -138,13 +144,12 @@ describe('persisted Fleet operations', () => {
|
||||
}
|
||||
}
|
||||
})();
|
||||
// The former OR query is the compatibility reference for all three branches.
|
||||
// Completed work, including warnings and cancellations, belongs to server history.
|
||||
const expected = handle.prepare(`SELECT d.id FROM server_deployments d
|
||||
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
||||
OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
|
||||
OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
|
||||
OR (d.state IN ('failed', 'interrupted') AND NOT EXISTS
|
||||
(SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
|
||||
.all(since) as { id: string }[];
|
||||
.all() as { id: string }[];
|
||||
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
expect(fleetOperations(now).map(row => row.key).sort()).toEqual(expected.map(row => `deployment:${row.id}`).sort());
|
||||
});
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const now = Date.parse('2026-10-05T12:00:00Z');
|
||||
const iso = (offset = 0) => new Date(now + offset).toISOString();
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now);
|
||||
fixture = await newFixture([]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().transaction(() => {
|
||||
const server = db().prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)');
|
||||
const operation = db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by)
|
||||
VALUES(?,?,'restart',?,'Complete',?,?,'test')`);
|
||||
for (const name of ['one', 'two']) {
|
||||
server.run(name, name);
|
||||
for (let i = 0; i < 100; i++) operation.run(`${name}-${i}`, name, 'done', iso(-i), iso(-i));
|
||||
operation.run(`${name}-old`, name, 'done', iso(-86_400_001), iso(-86_400_001));
|
||||
operation.run(`${name}-failed`, name, 'failed', iso(), iso());
|
||||
operation.run(`${name}-active`, name, 'waiting', iso(), null);
|
||||
}
|
||||
db().prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','push',?,?)").run(iso(), iso());
|
||||
for (const name of ['one', 'two']) {
|
||||
db().prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,checked_at)
|
||||
VALUES('campaign',?,1,'done',?)`).run(name, iso());
|
||||
}
|
||||
})();
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.restoreAllMocks(); });
|
||||
|
||||
describe('fleet and server response scope', () => {
|
||||
it('omits completed history from fleet REST and SSR responses', async () => {
|
||||
const { GET } = await import('../../src/routes/api/fleet/+server.js');
|
||||
const { load } = await import('../../src/routes/+page.server.js');
|
||||
const body = await GET().json();
|
||||
expect(body.operations.map((operation: { key: string }) => operation.key).sort())
|
||||
.toEqual(['deployment:one-active', 'deployment:two-active']);
|
||||
expect(load().fleetSnapshot.operations).toEqual(body.operations);
|
||||
});
|
||||
|
||||
it('loads only the selected server and its recent history in both REST and SSR', async () => {
|
||||
const { GET } = await import('../../src/routes/api/servers/[name]/+server.js');
|
||||
const { load } = await import('../../src/routes/servers/[name]/+page.server.js');
|
||||
const registry = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { fleetSnapshot } = await import('../../src/lib/server/fleet/snapshot.js');
|
||||
const expected = fleetSnapshot().servers.find(server => server.name === 'one');
|
||||
const list = vi.spyOn(registry, 'listServers');
|
||||
const snapshot = load({ params: { name: 'one' } }).serverSnapshot;
|
||||
expect(list).not.toHaveBeenCalled();
|
||||
expect(snapshot.servers).toEqual([expected]);
|
||||
expect(snapshot.operations).toHaveLength(103);
|
||||
expect(snapshot.operations.every(operation => operation.server === 'one')).toBe(true);
|
||||
expect(snapshot.operations.map(operation => operation.key)).not.toContain('deployment:one-old');
|
||||
expect(snapshot.operations).toContainEqual(expect.objectContaining({ key: 'campaign:campaign:one', state: 'done' }));
|
||||
expect(snapshot.operations).toContainEqual(expect.objectContaining({ key: 'deployment:one-failed', state: 'attention' }));
|
||||
const body = await GET({ params: { name: 'one' } }).json();
|
||||
expect(body).toEqual(JSON.parse(JSON.stringify(snapshot)));
|
||||
expect(list).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns no unrelated servers or history for missing and removed servers', async () => {
|
||||
const { serverSnapshot } = await import('../../src/lib/server/fleet/snapshot.js');
|
||||
expect(serverSnapshot('missing')).toMatchObject({ servers: [], operations: [] });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE servers SET destroyed_at = ? WHERE name = 'one'").run(iso());
|
||||
expect(serverSnapshot('one').servers).toEqual([]);
|
||||
db().prepare("UPDATE server_deployments SET server_deleted_at = ? WHERE server = 'one'").run(iso());
|
||||
db().prepare("UPDATE deployment_servers SET server_deleted_at = ? WHERE server = 'one'").run(iso());
|
||||
expect(serverSnapshot('one').operations).toEqual([]);
|
||||
});
|
||||
|
||||
it('uses a server index for recent lifecycle history', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { serverOperations } = await import('../../src/lib/server/fleet/overview.js');
|
||||
const prepare = vi.spyOn(db(), 'prepare');
|
||||
serverOperations('one', now);
|
||||
const sql = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM server_deployments'))!;
|
||||
prepare.mockRestore();
|
||||
const plan = db().prepare(`EXPLAIN QUERY PLAN ${sql}`).all({ server: 'one', since: iso(-86_400_000) }) as { detail: string }[];
|
||||
const details = plan.map(row => row.detail).join('\n');
|
||||
expect(details).toMatch(/SEARCH server_deployments USING (?:COVERING )?INDEX idx_server_deployments_server/);
|
||||
expect(details).not.toMatch(/SCAN server_deployments\b/);
|
||||
});
|
||||
});
|
||||
@@ -70,10 +70,10 @@ describe('provisioning reads with retained lifecycle history', () => {
|
||||
expect(queries).toHaveLength(1);
|
||||
const statement = handle.prepare(queries[0]);
|
||||
expect(statement.columns().map(column => column.name)).toEqual(['id', 'server', 'action', 'state', 'stage', 'error', 'started_at']);
|
||||
const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(iso(-86_400_000)) as { detail: string }[];
|
||||
const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[];
|
||||
const details = plan.map(row => row.detail).join('\n');
|
||||
expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_state');
|
||||
expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_finished_at');
|
||||
expect(queries[0]).not.toContain('finished_at >=');
|
||||
expect(details).toContain('SEARCH server_deployments USING INTEGER PRIMARY KEY (rowid=?)');
|
||||
expect(details).toContain('SCAN server_deployments USING COVERING INDEX idx_server_deployments_server');
|
||||
expect(details).not.toMatch(/SCAN server_deployments(?:\n|$)/);
|
||||
|
||||
Reference in New Issue
Block a user