Clarify host readiness when game files or runtime are missing

This commit is contained in:
2026-10-04 10:03:43 -04:00
parent 3d62dfff27
commit 62d7817ea3
8 changed files with 58 additions and 10 deletions
+4 -1
View File
@@ -23,7 +23,10 @@ the agent owns local execution and its durable operation journal.
no separate DLC declaration. Uploading also works before enrolling any host. no separate DLC declaration. Uploading also works before enrolling any host.
5. Cache the published version on each destination host. Wait for verification. 5. Cache the published version on each destination host. Wait for verification.
Hosts report connection and readiness separately; deployment also checks the Hosts report connection and readiness separately; deployment also checks the
particular library, image, capacity and ports you selected. particular library, image, capacity and ports you selected. A connected host
missing content shows **Needs runtime** or **Needs game files**; these require
setup rather than waiting for an automatic preparation step. **Ready** means
it has both a runtime image and a verified library available.
6. Launch a server or collection. Select the host, immutable game-file version(s) 6. Launch a server or collection. Select the host, immutable game-file version(s)
and runtime image. EC selects lobby IDs and reserves UDP/TCP ports atomically. and runtime image. EC selects lobby IDs and reserves UDP/TCP ports atomically.
Saved Plutonium server keys remain associated with their exact server names. Saved Plutonium server keys remain associated with their exact server names.
+6 -1
View File
@@ -1,7 +1,12 @@
export type HostMode = 'active' | 'draining' | 'disabled'; export type HostMode = 'active' | 'draining' | 'disabled';
export type HostReadiness = 'preparing' | 'ready' | 'degraded' | 'draining' | 'disabled'; export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled';
export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown'; export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export function hostReadinessLabel(readiness: HostReadiness): string {
return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled' }[readiness];
}
export interface HostDeletion { allowed: boolean; blockers: string[] } export interface HostDeletion { allowed: boolean; blockers: string[] }
export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string } export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
+5 -3
View File
@@ -23,14 +23,16 @@ function hostFromRow(row: HostRow): Host {
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.'); if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.'); if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.');
const infrastructureReady = reasons.length === 0; const infrastructureReady = reasons.length === 0;
if (!inventory.runtimeImages?.length) reasons.push('Prepare a Plutainer runtime image.'); const runtimeReady = !!inventory.runtimeImages?.length;
if (!inventory.libraries?.some(library => library.state === 'ready')) reasons.push('Download and verify a game-file library.'); const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready');
if (!runtimeReady) reasons.push('Prepare a Plutainer runtime image.');
if (!libraryReady) reasons.push('Download and verify a game-file library.');
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.'); if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
if (row.mode === 'disabled') reasons.push('Host is disabled.'); if (row.mode === 'disabled') reasons.push('Host is disabled.');
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max, return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes, bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline', createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at || (infrastructureReady && reasons.length) ? 'preparing' : reasons.length ? 'degraded' : 'ready', readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active }; readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
} }
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); } export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
@@ -1,6 +1,7 @@
<script lang="ts"> <script lang="ts">
import { onMount } from 'svelte'; import { onMount } from 'svelte';
import type { Host, NativeDeploymentSelection } from '$lib/hosts.js'; import type { Host, NativeDeploymentSelection } from '$lib/hosts.js';
import { hostReadinessLabel } from '$lib/hosts.js';
import type { GameLibrary } from '$lib/game-files.js'; import type { GameLibrary } from '$lib/game-files.js';
import type { ZombieGame } from '$lib/map-aliases.js'; import type { ZombieGame } from '$lib/map-aliases.js';
let { games, selection = $bindable(), ready = $bindable(false), disabled = false, network = false, onchange }: let { games, selection = $bindable(), ready = $bindable(false), disabled = false, network = false, onchange }:
@@ -26,7 +27,7 @@
<div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div> <div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div>
{#if error}<p role="alert" class="error">{error}</p>{/if} {#if error}<p role="alert" class="error">{error}</p>{/if}
{#if loading}<p role="status">Loading host readiness and libraries…</p>{/if} {#if loading}<p role="status">Loading host readiness and libraries…</p>{/if}
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {candidate.readiness}</option>{/each}</select></label> <label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {hostReadinessLabel(candidate.readiness)}</option>{/each}</select></label>
{#if !loading && !hosts.some(candidate => candidate.readiness === 'ready')}<p>No deployment-ready hosts. <a href="/hosts" target="_blank" rel="noopener">Set up a host</a>, then refresh.</p>{/if} {#if !loading && !hosts.some(candidate => candidate.readiness === 'ready')}<p>No deployment-ready hosts. <a href="/hosts" target="_blank" rel="noopener">Set up a host</a>, then refresh.</p>{/if}
{#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if} {#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if}
{#each games as game}<label>{game === 't5' ? 'Black Ops I' : 'Black Ops II'} game-file version<select bind:value={selection.libraryIds[game]} disabled={disabled || loading || !host} onchange={() => onchange?.()}><option value="">Choose an immutable library version</option>{#each versions.filter(version => version.game === game) as version}<option value={version.id} disabled={!cacheReady(version.id)}>{version.name} · {version.version} · {cacheReady(version.id) ? 'Ready on host' : 'Not cached on host'}</option>{/each}</select></label>{/each} {#each games as game}<label>{game === 't5' ? 'Black Ops I' : 'Black Ops II'} game-file version<select bind:value={selection.libraryIds[game]} disabled={disabled || loading || !host} onchange={() => onchange?.()}><option value="">Choose an immutable library version</option>{#each versions.filter(version => version.game === game) as version}<option value={version.id} disabled={!cacheReady(version.id)}>{version.name} · {version.version} · {cacheReady(version.id) ? 'Ready on host' : 'Not cached on host'}</option>{/each}</select></label>{/each}
+2 -1
View File
@@ -2,6 +2,7 @@
import { onMount } from 'svelte'; import { onMount } from 'svelte';
import { page } from '$app/stores'; import { page } from '$app/stores';
import type { Host } from '$lib/hosts.js'; import type { Host } from '$lib/hosts.js';
import { hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte'; import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -50,7 +51,7 @@
{#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if} {#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
</section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if} </section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
{#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else} {#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
<div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{host.readiness}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>Manage host</Button>{#if canManage}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>Delete host</Button>{/if}</div></article>{/each}</div> <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>Manage host</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>Delete host</Button>{/if}</div></article>{/each}</div>
{/if} {/if}
</section> </section>
<DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} /> <DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />
+3 -1
View File
@@ -3,6 +3,7 @@
import { page } from '$app/stores'; import { page } from '$app/stores';
import { goto } from '$app/navigation'; import { goto } from '$app/navigation';
import type { Host, HostOperation, HostMode } from '$lib/hosts.js'; import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
import { hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte'; import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -36,8 +37,9 @@
</SectionHeader> </SectionHeader>
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if} {#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if}
{#if loading}<p role="status">Loading host…</p>{:else if host} {#if loading}<p role="status">Loading host…</p>{:else if host}
<section class="panel"><div class="row"><h2>{host.connectionState} · {host.readiness}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div> <section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
{#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if} {#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if}
{#if host.readiness === 'needs-game-files'}<p><a href="/game-files">Choose a published game-file version and download it to this host →</a></p>{/if}
{#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if} {#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
<p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p> <p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
</section> </section>
+1 -1
View File
@@ -156,7 +156,7 @@ describe('host deletion', () => {
expect(roleAllows('admin', permission)).toBe(true); expect(roleAllows('admin', permission)).toBe(true);
expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false); expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false);
const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js'); const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js');
for (const headers of [{ origin: 'https://attacker.test' }, { authorization: `Bearer ${hosts.credentials.credential}` }]) { for (const headers of [new Headers({ origin: 'https://attacker.test' }), new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })]) {
expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 })); expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 }));
} }
expect(hosts.getHost(hosts.host.id)).not.toBeNull(); expect(hosts.getHost(hosts.host.id)).not.toBeNull();
+35 -1
View File
@@ -71,7 +71,7 @@ describe('native host readiness and reservations', () => {
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready'); expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime'); expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime');
expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library'); expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library');
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('preparing'); expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('needs-game-files');
const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false }); const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false });
expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded'); expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded');
expect(() => api.requireReadyHost(host.id)).toThrow('Docker'); expect(() => api.requireReadyHost(host.id)).toThrow('Docker');
@@ -79,6 +79,40 @@ describe('native host readiness and reservations', () => {
vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000); vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000);
expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale'); expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale');
}); });
it.each([
{ missing: { libraries: [] }, expected: 'needs-game-files' },
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
])('names missing content as $expected until a prepared heartbeat arrives', async ({ missing, expected }) => {
const api = await enrolled();
const host = api.heartbeat(api.host.id, { ...inventory, ...missing });
expect(host.connected).toBe(true);
expect(host.readiness).toBe(expected);
expect(() => api.requireReadyHost(host.id)).toThrow();
expect(api.heartbeat(host.id, inventory).readiness).toBe('ready');
expect(api.requireReadyHost(host.id).id).toBe(host.id);
});
it('keeps setup work deliverable while new instance creation waits for missing game files', async () => {
const api = await enrolled();
const create = api.queueHostOperation(api.host.id, 'instance.create', {});
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('game-file library');
const download = api.queueHostOperation(api.host.id, 'library.download', {});
const runtime = api.queueHostOperation(api.host.id, 'runtime.pull', {});
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
api.heartbeat(api.host.id, inventory);
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(create.id);
});
it('gives enrollment, infrastructure failure and host modes priority over missing content', async () => {
const api = await enrolled();
expect(api.createHost({ name: 'New host' }).readiness).toBe('preparing');
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [], dockerRunning: false }).readiness).toBe('degraded');
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
expect(api.updateHost(api.host.id, { mode: 'draining' }).readiness).toBe('draining');
expect(api.updateHost(api.host.id, { mode: 'disabled' }).readiness).toBe('disabled');
});
it('blocks new deployments on a draining or disabled host without altering existing operations', async () => { it('blocks new deployments on a draining or disabled host without altering existing operations', async () => {
const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 }); const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 });
api.updateHost(api.host.id, { mode: 'draining' }); api.updateHost(api.host.id, { mode: 'draining' });