Show provisioning status during server creation

This commit is contained in:
2026-09-19 22:53:24 -04:00
parent 5985959882
commit 44b6415821
12 changed files with 72 additions and 12 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
import { freshStatusTimestamp } from '../../server-freshness.js'; import { freshStatusTimestamp } from '../../server-freshness.js';
export type Health = 'online' | 'stale' | 'offline' | 'starting'; export type Health = 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
const STALE_MAX_MS = 120_000; const STALE_MAX_MS = 120_000;
+9
View File
@@ -0,0 +1,9 @@
import { db } from '../db/index.js';
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return operation?.action === 'create' && ['queued', 'running', 'awaiting-health'].includes(operation.state);
}
+13
View File
@@ -9,6 +9,7 @@ import { getServer, listServers, setServerField } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js'; import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js'; import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js'; import { classifyHealth } from '../fleet/health.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js';
const TICK_MS = 15_000; const TICK_MS = 15_000;
@@ -74,6 +75,18 @@ async function refreshStatus(): Promise<void> {
const i = cursor++; const i = cursor++;
if (i >= servers.length) return; if (i >= servers.length) return;
const srv = servers[i]; const srv = servers[i];
// Creation performs its own container, RCON, and map verification.
if (isProvisioning(srv.name)) {
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: 'provisioning', lastSeenAt: srv.last_seen_at,
lastError: null, endpoint: null, reason: null, replySnippet: null,
activeSince: null, inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const container = restartSettling(srv.name) const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' } ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch((err) => ({ : await komodoInspectHealthy(srv.name).catch((err) => ({
+3 -1
View File
@@ -107,7 +107,9 @@
<StatusPip {health} /> <StatusPip {health} />
<strong class="name"><ServerIdentity {name} mapLabel={mapLabelOverride} /></strong> <strong class="name"><ServerIdentity {name} mapLabel={mapLabelOverride} /></strong>
</div> </div>
{#if health === 'starting'} {#if health === 'provisioning'}
<Badge variant="info" uppercase>Provisioning</Badge>
{:else if health === 'starting'}
<Badge variant="info" uppercase>Starting</Badge> <Badge variant="info" uppercase>Starting</Badge>
{:else if health === 'offline'} {:else if health === 'offline'}
<Badge variant="danger" uppercase>offline</Badge> <Badge variant="danger" uppercase>offline</Badge>
+1 -1
View File
@@ -26,7 +26,7 @@
background: var(--spark-500); background: var(--spark-500);
animation: electric-pulse 1.6s ease-in-out infinite; animation: electric-pulse 1.6s ease-in-out infinite;
} }
.dot-stale, .dot-starting { .dot-stale, .dot-starting, .dot-provisioning {
background: var(--amber-500); background: var(--amber-500);
box-shadow: var(--amber-glow); box-shadow: var(--amber-glow);
} }
@@ -11,7 +11,7 @@
let toneClass = $derived( let toneClass = $derived(
health === 'online' ? 't-online' : health === 'online' ? 't-online' :
health === 'stale' || health === 'starting' ? 't-warning' : health === 'stale' || health === 'starting' || health === 'provisioning' ? 't-warning' :
't-danger' 't-danger'
); );
@@ -19,6 +19,7 @@
label ?? ( label ?? (
health === 'online' ? 'online' : health === 'online' ? 'online' :
health === 'starting' ? 'starting' : health === 'starting' ? 'starting' :
health === 'provisioning' ? 'provisioning' :
health === 'stale' ? 'stale' : health === 'stale' ? 'stale' :
'offline' 'offline'
) )
+1
View File
@@ -1,5 +1,6 @@
export type StatusTone = 'neutral' | 'online' | 'warning' | 'danger' | 'info'; export type StatusTone = 'neutral' | 'online' | 'warning' | 'danger' | 'info';
const states: Record<string, [string, StatusTone]> = { const states: Record<string, [string, StatusTone]> = {
provisioning: ['Provisioning', 'info'],
online: ['Online', 'online'], offline: ['Offline', 'danger'], stale: ['Report outdated', 'warning'], starting: ['Starting', 'info'], online: ['Online', 'online'], offline: ['Offline', 'danger'], stale: ['Report outdated', 'warning'], starting: ['Starting', 'info'],
queued: ['Queued', 'neutral'], pending: ['Queued for restart', 'neutral'], running: ['In progress', 'info'], active: ['In progress', 'info'], queued: ['Queued', 'neutral'], pending: ['Queued for restart', 'neutral'], running: ['In progress', 'info'], active: ['In progress', 'info'],
deploying: ['Installing files', 'info'], restarting: ['Restarting', 'info'], deploying: ['Installing files', 'info'], restarting: ['Restarting', 'info'],
+1 -1
View File
@@ -13,7 +13,7 @@ export interface RosterPlayer {
name: string; name: string;
} }
export type Health = 'online' | 'stale' | 'offline' | 'starting'; export type Health = 'online' | 'stale' | 'offline' | 'starting' | 'provisioning';
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players'; export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
export interface RconEndpoint { export interface RconEndpoint {
+3 -2
View File
@@ -86,6 +86,7 @@
let empty = $derived(healthy.filter(server => server.last_players === 0).length); let empty = $derived(healthy.filter(server => server.last_players === 0).length);
let problems = $derived(servers.filter(server => server.health === 'offline' || server.health === 'stale')); let problems = $derived(servers.filter(server => server.health === 'offline' || server.health === 'stale'));
let starting = $derived(servers.filter(server => server.health === 'starting').length); let starting = $derived(servers.filter(server => server.health === 'starting').length);
let provisioning = $derived(servers.filter(server => server.health === 'provisioning').length);
let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, $uiNow)) let filtered = $derived(servers.filter(server => matchesFleet(server, query, filter, game, pinned, activeOperations, $uiNow))
.sort((a, b) => { .sort((a, b) => {
const pinOrder = Number(pinned.includes(b.name)) - Number(pinned.includes(a.name)); const pinOrder = Number(pinned.includes(b.name)) - Number(pinned.includes(a.name));
@@ -161,13 +162,13 @@
<div class="setup-note"><span><strong>Finish system setup</strong> · {!$fleetIntegrations.komodo ? 'Komodo' : ''}{!$fleetIntegrations.komodo && !$fleetIntegrations.rcon ? ' and ' : ''}{!$fleetIntegrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div> <div class="setup-note"><span><strong>Finish system setup</strong> · {!$fleetIntegrations.komodo ? 'Komodo' : ''}{!$fleetIntegrations.komodo && !$fleetIntegrations.rcon ? ' and ' : ''}{!$fleetIntegrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
{/if} {/if}
{#if problems.length || attentionOperations.length || starting} {#if problems.length || attentionOperations.length || starting || provisioning}
<div class="attention" class:has-issues={problems.length > 0 || attentionOperations.length > 0}> <div class="attention" class:has-issues={problems.length > 0 || attentionOperations.length > 0}>
<span class="attention-mark" aria-hidden="true">{problems.length || attentionOperations.length ? '!' : '✓'}</span> <span class="attention-mark" aria-hidden="true">{problems.length || attentionOperations.length ? '!' : '✓'}</span>
<div class="attention-copy"> <div class="attention-copy">
{#if problems.length} {#if problems.length}
<button onclick={() => { resetFilters(); filter = 'attention'; }}>{problems.length} {problems.length === 1 ? 'server needs' : 'servers need'} attention →</button> <button onclick={() => { resetFilters(); filter = 'attention'; }}>{problems.length} {problems.length === 1 ? 'server needs' : 'servers need'} attention →</button>
{:else}<strong>{servers.length ? starting ? `${starting} ${starting === 1 ? 'server is' : 'servers are'} starting` : 'All servers healthy' : 'Preparing your fleet'}</strong>{/if} {:else}<strong>{provisioning ? `${provisioning} ${provisioning === 1 ? 'server is' : 'servers are'} provisioning` : servers.length ? starting ? `${starting} ${starting === 1 ? 'server is' : 'servers are'} starting` : 'All servers healthy' : 'Preparing your fleet'}</strong>{/if}
</div> </div>
{#if attentionOperations.length}<a href="#fleet-operations" onclick={() => operationsOpen = true}>{attentionOperations.length} operation {attentionOperations.length === 1 ? 'needs' : 'need'} attention →</a>{/if} {#if attentionOperations.length}<a href="#fleet-operations" onclick={() => operationsOpen = true}>{attentionOperations.length} operation {attentionOperations.length === 1 ? 'needs' : 'need'} attention →</a>{/if}
+4 -2
View File
@@ -3,6 +3,7 @@ import { kick } from '$lib/server/pipeline/worker.js';
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { listServers, heldByBeta } from '$lib/server/fleet/servers.js'; import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
import { classifyHealth } from '$lib/server/fleet/health.js'; import { classifyHealth } from '$lib/server/fleet/health.js';
import { isProvisioning } from '$lib/server/fleet/provisioning.js';
import { queueCreateServer } from '$lib/server/fleet/create.js'; import { queueCreateServer } from '$lib/server/fleet/create.js';
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js'; import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
import type { RconPlayer } from '$lib/server/integrations/rcon.js'; import type { RconPlayer } from '$lib/server/integrations/rcon.js';
@@ -20,7 +21,8 @@ function safeJson<T>(raw: string | null): T | null {
export async function GET() { export async function GET() {
const rconHost = config.rcon.host; const rconHost = config.rcon.host;
const servers = listServers().map((s) => { const servers = listServers().map((s) => {
const ready = !s.container_state || s.container_state === 'running'; const provisioning = isProvisioning(s.name);
const ready = !provisioning && (!s.container_state || s.container_state === 'running');
return { return {
...s, ...s,
last_players: ready ? s.last_players : null, last_players: ready ? s.last_players : null,
@@ -30,7 +32,7 @@ export async function GET() {
game_started_at: ready ? s.game_started_at : null, game_started_at: ready ? s.game_started_at : null,
game_ended_at: ready ? s.game_ended_at : null, game_ended_at: ready ? s.game_ended_at : null,
roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null, roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null,
health: classifyHealth(s.last_seen_at, s.last_error, s.container_state), health: provisioning ? 'provisioning' : classifyHealth(s.last_seen_at, s.last_error, s.container_state),
replySnippet: ready ? s.last_reply_snippet : null, replySnippet: ready ? s.last_reply_snippet : null,
// Derived endpoint — same logic as endpointFor() but without asking // Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null. // Komodo. If we don't have a port yet, endpoint is null.
+6 -3
View File
@@ -160,7 +160,7 @@
let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince); let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince);
let gameElapsed = $derived(fresh && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null); let gameElapsed = $derived(fresh && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting'); let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting' && effectiveHealth !== 'provisioning');
function ago(iso: string | null, now: number): string { function ago(iso: string | null, now: number): string {
if (!iso) return 'never'; if (!iso) return 'never';
@@ -243,6 +243,7 @@
<span class="health-label"> <span class="health-label">
{effectiveHealth === 'online' ? 'Online' : {effectiveHealth === 'online' ? 'Online' :
effectiveHealth === 'starting' ? 'Starting' : effectiveHealth === 'starting' ? 'Starting' :
effectiveHealth === 'provisioning' ? 'Provisioning' :
effectiveHealth === 'stale' ? 'Stale' : 'Offline'} effectiveHealth === 'stale' ? 'Stale' : 'Offline'}
</span> </span>
</div> </div>
@@ -286,7 +287,7 @@
{/if} {/if}
</div> </div>
{#if effectiveHealth !== 'starting'}<RoundTelemetry {name} />{/if} {#if effectiveHealth !== 'starting' && effectiveHealth !== 'provisioning'}<RoundTelemetry {name} />{/if}
</Card> </Card>
@@ -348,7 +349,9 @@
<Card padding="lg"> <Card padding="lg">
<div class="panel-head"> <div class="panel-head">
<h2>RCON diagnostic</h2> <h2>RCON diagnostic</h2>
{#if effectiveHealth === 'starting'} {#if effectiveHealth === 'provisioning'}
<Badge variant="info" uppercase>Provisioning</Badge>
{:else if effectiveHealth === 'starting'}
<Badge variant="info" uppercase>Waiting for container</Badge> <Badge variant="info" uppercase>Waiting for container</Badge>
{:else if hasDiagnosticIssue} {:else if hasDiagnosticIssue}
<Badge variant="warning" uppercase>Issue</Badge> <Badge variant="warning" uppercase>Issue</Badge>
+28
View File
@@ -30,6 +30,34 @@ async function setup() {
} }
describe('database fleet and container readiness', () => { describe('database fleet and container readiness', () => {
it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => {
const { db, inspect, rcon, fanoutStatus } = await setup();
const { GET } = await import('../../src/routes/api/servers/+server.js');
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
db.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')`).run(state, new Date().toISOString());
const snapshot = async () => (await (await GET()).json()).servers[0];
expect(await snapshot()).toMatchObject({ health: 'provisioning', last_error: null, last_players: null });
const events: unknown[] = [];
const unsubscribe = subscribe(event => events.push(event));
try { await fanoutStatus(); } finally { unsubscribe(); }
expect(inspect).not.toHaveBeenCalled();
expect(rcon).not.toHaveBeenCalled();
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
health: 'provisioning', lastError: null, reason: null, replySnippet: null
})] }));
db.prepare("UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'").run();
inspect.mockResolvedValue({ ok: true, state: 'running' });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'offline', last_error: 'timeout:no reply' });
db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run();
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null });
});
it('keeps game_end through occupied polls until game_start or an empty lobby', async () => { it('keeps game_end through occupied polls until game_start or an empty lobby', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { inspect, rcon, fanoutStatus, getServer } = await setup();
inspect.mockResolvedValue({ ok: true, state: 'running' }); inspect.mockResolvedValue({ ok: true, state: 'running' });