josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Cache Plutonium runtime files on hosts

Browse Source
This commit is contained in:
josh
2026-10-07 15:17:59 -04:00
parent 7fd59d32b4
commit 062ade8655
13 changed files with 1418 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/hosts.ts
+2 -1
View File
@@ -82,6 +82,7 @@ export interface HostInventory {
directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
currentOperation?: string | null; [key: string]: unknown;
capabilities?: string[];
plutoniumCache?: { revision: number; bytes: number };
agentUpdate?: HostAgentUpdate;
agentLanes?: HostAgentLane[];
instanceResources?: InstanceResources[];
@@ -146,7 +147,7 @@ export interface HostOperationHistoryPage {
}
export interface HostPreparationItem {
kind: 'runtime' | 'library'; label: string;
kind: 'runtime' | 'library' | 'plutonium'; label: string;
state: 'waiting' | 'preparing' | 'ready' | 'blocked'; detail: string;
operationId?: string; bytesDownloaded?: number; bytesTotal?: number;
cache?: HostLibraryCache;
src/lib/server/hosts/index.ts
+19 -6
View File
@@ -277,6 +277,12 @@ export function validateInventory(value: unknown): HostInventory {
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
if (inventory.plutoniumCache !== undefined) {
const cache = inventory.plutoniumCache;
if (!cache || typeof cache !== 'object' || Array.isArray(cache) || !Number.isSafeInteger(cache.revision) || cache.revision < 1
|| !Number.isSafeInteger(cache.bytes) || cache.bytes < 0) throw new Error('Invalid Plutonium cache inventory.');
inventory.plutoniumCache = { revision: cache.revision, bytes: cache.bytes };
}
if (inventory.agentUpdate !== undefined) {
const update = inventory.agentUpdate;
const version = (value: unknown) => typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value);
@@ -396,6 +402,14 @@ export type OperationDeliveryRow = Omit<OperationRow, 'payload'>;
export type HostOperationStatus = Omit<HostOperation, 'payload'>;
const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
const readOnlyOperationSql = `(o.kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
OR (o.kind='rcon.execute' AND COALESCE(json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version'),0)))`;
/** Optional cache downloads yield their write lane to deployments and host changes. */
export function shouldDeferPlutoniumPreparation(hostId: string): boolean {
return !!database().prepare(`SELECT 1 FROM ec_host_operations o WHERE o.host_id=?
AND o.state IN ('pending','accepted','running','unknown') AND o.kind!='plutonium.prepare'
AND NOT ${readOnlyOperationSql} LIMIT 1`).get(hostId);
}
function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; }
export function getHostOperation(id: string): HostOperation | null {
@@ -483,7 +497,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
}
}
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','plutonium.prepare','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
const requestHash = createHash('sha256').update(payload).digest('hex');
const packed = packFileJson(JSON.parse(payload.toString()));
payload = JSON.stringify(packed.value);
@@ -523,12 +537,10 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
// Health never queues behind a delivered file batch. Keep background reads
// in small batches to amortize delivery without handing out a long backlog.
const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
@@ -536,13 +548,14 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','plutonium.prepare','host.inspect')" : ''}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''}
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END,
COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
// A short replay lease avoids handing the idle health lane its busy
// sibling's work, but still recovers a lost receipt or agent restart.
.all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),
src/lib/server/hosts/observations.ts
+4
View File
@@ -35,6 +35,10 @@ export const importantOperationSql = `(o.routine=0 OR o.state IN ('failed','unkn
const payloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
/** Deliberately exclude arbitrary RCON commands and all file contents/credentials. */
export const operationTargetSql = `CASE
WHEN o.kind='plutonium.prepare' THEN CASE
WHEN json_type(${payloadSql},'$.revision')='integer' AND json_extract(${payloadSql},'$.revision')>0
THEN 'Production revision r' || json_extract(${payloadSql},'$.revision')
ELSE 'Latest production revision' END
WHEN o.kind IN ('files.read','files.stat','files.list','logs.read') THEN
CASE WHEN json_type(${payloadSql},'$.path')='text' THEN substr(json_extract(${payloadSql},'$.path'),1,1024) END
WHEN o.kind='files.manifest' THEN substr((SELECT group_concat(value,', ') FROM json_each(${payloadSql},'$.roots') WHERE type='text'),1,1024)
src/lib/server/hosts/preparation.ts
+51 -21
View File
@@ -1,15 +1,17 @@
import { db } from '../db/index.js';
import { config } from '../config.js';
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js';
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation, shouldDeferPlutoniumPreparation } from './index.js';
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
import { scheduleAvailabilityRefresh } from './availability.js';
import { readSetting } from '../maintenance/settings.js';
import type { PlutoniumRelease } from '../../plutonium.js';
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; revision?: number; requiredBytes: number; operation?: PreparationOperation };
type QueuedSelection = { libraryId: string };
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -48,25 +50,27 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
return [...queued.values(), ...latest];
}
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
function latestOperation(hostId: string, kind: string, field?: string, value?: string): PreparationOperation | undefined {
return db().prepare(`SELECT id,rowid AS sequence,last_delivered_at,state,updated_at,error FROM ec_host_operations
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
.get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
WHERE host_id=? AND kind=? ${field ? 'AND json_extract(payload,?)=?' : ''} ORDER BY created_at DESC,rowid DESC LIMIT 1`)
.get(hostId, kind, ...(field ? [`$.${field}`, value!] : [])) as PreparationOperation | undefined;
}
function retryAt(hostId: string, target: Target): number {
const operation = target.operation;
if (!operation || operation.state !== 'failed') return 0;
const kind = target.image ? 'runtime.pull' : 'library.download';
const field = target.image ? '$.image' : '$.libraryVersionId';
const value = target.image ?? target.versionId!;
const kind = target.image ? 'runtime.pull' : target.versionId ? 'library.download' : 'plutonium.prepare';
const field = target.image ? '$.image' : target.versionId ? '$.libraryVersionId' : null;
const value = target.image ?? target.versionId;
const match = field ? 'AND json_extract(payload,?)=?' : '';
const bindings = [hostId, kind, ...(field ? [field, value!] : [])];
// Count consecutive failures, including across controller restarts. Unknown
// operations are never retried under a new ID: their journal must reconcile.
const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='failed'
WHERE host_id=? AND kind=? ${match} AND state='failed'
AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='completed'),0)`)
.get(hostId, kind, field, value, hostId, kind, field, value) as { count: number };
WHERE host_id=? AND kind=? ${match} AND state='completed'),0)`)
.get(...bindings, ...bindings) as { count: number };
return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
}
@@ -108,6 +112,22 @@ function targets(host: Host): Target[] {
: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } });
}
if (host.inventory.capabilities?.includes('plutonium.cache')) {
// Read the same production release evidence as freshRelease without
// importing its fleet/provisioning observers into host preparation.
const release = readSetting<PlutoniumRelease>('plutonium_release', { revision: null, checkedAt: null, error: null });
const age = release.checkedAt ? Date.now() - Date.parse(release.checkedAt) : NaN;
const revision = !release.error && age >= 0 && age < 60 * 60_000
&& Number.isSafeInteger(release.revision) && release.revision! > 0 ? release.revision! : 0;
const cache = host.inventory.plutoniumCache;
const ready = !!cache && cache.revision >= revision;
result.push({ revision, requiredBytes: host.minimumFreeBytes,
operation: latestOperation(host.id, 'plutonium.prepare'),
item: { kind: 'plutonium', label: 'Plutonium cache (optional)', state: ready ? 'ready' : 'waiting',
detail: ready ? `Production revision r${cache.revision} is cached for new deployments.`
: cache ? `Production revision r${revision} will replace cached r${cache.revision} automatically.`
: 'Plutonium files will be cached after the runtime and game files are ready.' } });
}
const blocker = hostBlocker(host);
for (const target of result) {
if (target.item.state === 'ready') continue;
@@ -116,14 +136,15 @@ function targets(host: Host): Target[] {
if (blocker) { Object.assign(target.item, blocker); continue; }
if (operation && activeStates.includes(operation.state)) {
target.item.state = 'preparing';
target.item.detail = !target.image && operation.state === 'pending' && !operation.last_delivered_at
target.item.detail = target.versionId && operation.state === 'pending' && !operation.last_delivered_at
? 'Queued for game-file preparation; waiting for agent delivery or fleet preparation capacity.'
: operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
: target.image ? 'Downloading the runtime automatically.'
: target.revision !== undefined ? 'Downloading and verifying the optional Plutonium cache.'
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
} else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
} else if ((target.revision !== undefined || !enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
target.item.state = 'blocked';
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
@@ -142,16 +163,19 @@ export function hostPreparationStatus(host: Host): HostPreparation {
}
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
const state = items.some(item => item.state === 'blocked') ? 'blocked'
: items.some(item => item.state === 'preparing') ? 'preparing'
: items.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
// Cache health is visible, but never makes a usable host ineligible or
// delays re-enabling: a lobby's normal updater handles cache misses.
const required = items.filter(item => item.kind !== 'plutonium');
const state = required.some(item => item.state === 'blocked') ? 'blocked'
: required.some(item => item.state === 'preparing') ? 'preparing'
: required.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
return { state, items };
}
export function presentHostPreparation(host: Host): Host {
const preparation = hostPreparationStatus(host);
if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.state !== 'ready').map(item => item.detail), preparation };
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.kind !== 'plutonium' && item.state !== 'ready').map(item => item.detail), preparation };
}
return { ...host, preparation };
}
@@ -180,20 +204,26 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
if (!host) throw new Error('Host was not found.');
if (!hostBlocker(host)) {
supersedeUndeliveredDownloads(hostId);
// Queue only one setup mutation at a time so each capacity check uses
// fresh disk inventory. This also respects manually queued downloads.
// Queue one required setup mutation at a time with fresh disk
// inventory. A running optional cache yields to required work via
// the heartbeat flag, then resumes under its original operation ID.
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed')
AND kind IN ('runtime.pull','library.download','library.import','plutonium.prepare') AND state IN ('completed','failed')
AND (error IS NULL OR error != ?)
ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined;
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
// A failed or oversized library must not prevent other libraries
// from preparing. The runtime still comes first when eligible.
const target = targets(host).find(target => target.item.state === 'waiting');
const candidates = targets(host);
const requiredReady = candidates.every(target => target.item.kind === 'plutonium' || target.item.state === 'ready');
const target = candidates.find(target => target.item.state === 'waiting' && (target.item.kind !== 'plutonium' || requiredReady));
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
else if (target?.revision !== undefined && !shouldDeferPlutoniumPreparation(host.id)) queueHostOperation(host.id, 'plutonium.prepare', {
revision: target.revision, minimumFreeBytes: host.minimumFreeBytes, automatic: true
});
}
}
const status = hostPreparationStatus(host);
src/lib/ui/components/HostPreparation.svelte
+1
View File
@@ -7,6 +7,7 @@
<div class="preparation" aria-live="polite">
<p>The default runtime and the latest available T5 and T6 game files are prepared automatically. Re-enabling a host requires a fresh runtime pull and verification of both BO1 (T5) and BO2 (T6) libraries before it accepts deployments. Running servers keep their current files.</p>
<p>Supported hosts also cache Plutonium files to speed up new deployments. This optional cache refreshes when a new production release is detected; deployments can proceed while it is unavailable.</p>
{#if host.preparation}
<p class="status" class:blocked={host.preparation.state === 'blocked'}>Automatic setup: {host.preparation.state === 'ready' ? 'Ready' : host.preparation.state === 'preparing' ? 'In progress' : host.preparation.state === 'blocked' ? 'Needs attention' : 'Waiting'}</p>
<ul>