josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Cache Plutonium runtime files on hosts

Browse Source
This commit is contained in:
josh
2026-10-07 15:17:59 -04:00
parent 7fd59d32b4
commit 062ade8655
13 changed files with 1418 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+19 -6
View File
@@ -277,6 +277,12 @@ export function validateInventory(value: unknown): HostInventory {
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
if (inventory.plutoniumCache !== undefined) {
const cache = inventory.plutoniumCache;
if (!cache || typeof cache !== 'object' || Array.isArray(cache) || !Number.isSafeInteger(cache.revision) || cache.revision < 1
|| !Number.isSafeInteger(cache.bytes) || cache.bytes < 0) throw new Error('Invalid Plutonium cache inventory.');
inventory.plutoniumCache = { revision: cache.revision, bytes: cache.bytes };
}
if (inventory.agentUpdate !== undefined) {
const update = inventory.agentUpdate;
const version = (value: unknown) => typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value);
@@ -396,6 +402,14 @@ export type OperationDeliveryRow = Omit<OperationRow, 'payload'>;
export type HostOperationStatus = Omit<HostOperation, 'payload'>;
const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
const readOnlyOperationSql = `(o.kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
OR (o.kind='rcon.execute' AND COALESCE(json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version'),0)))`;
/** Optional cache downloads yield their write lane to deployments and host changes. */
export function shouldDeferPlutoniumPreparation(hostId: string): boolean {
return !!database().prepare(`SELECT 1 FROM ec_host_operations o WHERE o.host_id=?
AND o.state IN ('pending','accepted','running','unknown') AND o.kind!='plutonium.prepare'
AND NOT ${readOnlyOperationSql} LIMIT 1`).get(hostId);
}
function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; }
export function getHostOperation(id: string): HostOperation | null {
@@ -483,7 +497,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
}
}
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','plutonium.prepare','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
const requestHash = createHash('sha256').update(payload).digest('hex');
const packed = packFileJson(JSON.parse(payload.toString()));
payload = JSON.stringify(packed.value);
@@ -523,12 +537,10 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
// Health never queues behind a delivered file batch. Keep background reads
// in small batches to amortize delivery without handing out a long backlog.
const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
@@ -536,13 +548,14 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','plutonium.prepare','host.inspect')" : ''}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''}
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END,
COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
// A short replay lease avoids handing the idle health lane its busy
// sibling's work, but still recovers a lost receipt or agent restart.
.all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false),