josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove host files and uninstall agent when deleting hosts

Browse Source
This commit is contained in:
josh
2026-10-04 10:20:51 -04:00
parent 62d7817ea3
commit 020894b816
20 changed files with 1743 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/index.ts
+89 -14
View File
@@ -1,8 +1,8 @@
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
import { isIP } from 'node:net';
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import type { Host, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
export const AGENT_PROTOCOL_VERSION = 1;
export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null };
function decommissionRow(hostId: string): DecommissionRow | undefined {
return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined;
}
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
}
function hostFromRow(row: HostRow): Host {
const inventory: HostInventory = JSON.parse(row.inventory);
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
@@ -29,11 +37,13 @@ function hostFromRow(row: HostRow): Host {
if (!libraryReady) reasons.push('Download and verify a game-file library.');
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
if (row.mode === 'disabled') reasons.push('Host is disabled.');
const decommission = presentDecommission(decommissionRow(row.id));
if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
}
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
export function getHost(id: string): Host | null {
@@ -43,14 +53,18 @@ export function getHost(id: string): Host | null {
function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
export class HostDeletionError extends Error {
constructor(message: string, readonly status: 404 | 409, readonly blockers: string[] = []) { super(message); }
constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); }
}
/** Controller records can be removed only after all remote instance ownership is resolved. */
export function hostDeletionStatus(id: string): HostDeletion {
if (!getHost(id)) throw new HostDeletionError('Host not found.', 404);
const host = getHost(id);
if (!host) throw new HostDeletionError('Host not found.', 404);
if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
const handle = database();
const blockers: string[] = [];
if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.');
if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.');
const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[];
if (instances.length) {
const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
@@ -70,20 +84,68 @@ export function hostDeletionStatus(id: string): HostDeletion {
return { allowed: blockers.length === 0, blockers };
}
/** Retain references and audit history; deletion never sends remote cleanup commands. */
export function deleteHost(id: string): void {
/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
const handle = database();
handle.transaction(() => {
return handle.transaction(() => {
const host = getHost(id);
if (!host) throw new HostDeletionError('Host not found.', 404);
if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId };
const status = hostDeletionStatus(id);
if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
const timestamp = now();
const reason = 'Host deleted before this operation was delivered to the agent.';
const reason = 'Host deletion requested before this operation was delivered to the agent.';
handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id);
handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id);
handle.prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, id);
handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id);
handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
if (!host.enrolled) {
handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
return { deleted: true };
}
const operationId = randomUUID();
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
.run(id, operationId, digest(token), timestamp, timestamp);
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
return { deleted: false, operationId };
}).immediate();
}
export function receiveDecommissionReceipt(operationId: string, token: string, input: Record<string, unknown>): { acknowledged: true } {
if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
return database().transaction(() => {
const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined;
if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
if (row.stage === 'completed') return { acknowledged: true as const };
const operation = getHostOperation(operationId)!;
if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
const timestamp = now();
if (input.stage === 'failed') {
database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?")
.run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id);
return { acknowledged: true as const };
}
if (input.stage === 'cleaned') {
database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id);
return { acknowledged: true as const };
}
if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409);
database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId);
database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp);
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id);
database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id);
if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) {
database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
}
database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id);
return { acknowledged: true as const };
}).immediate();
}
function textField(value: unknown, label: string, max = 200): string {
@@ -102,6 +164,7 @@ export function createHost(input: Record<string, unknown>): Host {
}
export function updateHost(id: string, input: Record<string, unknown>): Host {
const host = requireHost(id);
if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
const mode = input.mode ?? host.mode;
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
@@ -117,7 +180,8 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
}
export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
requireHost(hostId);
const host = requireHost(hostId);
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.');
if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
@@ -143,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory {
}
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
!library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
return inventory;
@@ -169,7 +234,8 @@ export function authenticateCredential(token: string): Host | null {
return getHost(row.host_id);
}
export function revokeCredentials(hostId: string): void {
requireHost(hostId);
const host = requireHost(hostId);
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
}
export function rotateCredential(hostId: string, nextCredential?: string): string {
@@ -207,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row =
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
const host = requireHost(hostId);
if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
@@ -245,6 +313,7 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
requireHost(hostId);
const operation = getHostOperation(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
if (['completed','failed'].includes(operation.state)) return operation;
const state = input.state as HostOperationState;
@@ -253,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
const result = input.result === undefined ? null : JSON.stringify(input.result);
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
if (operation.kind === 'host.decommission') {
if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
}
if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
return getHostOperation(id)!;
})();
@@ -270,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
if (host.decommission) return false;
if (candidate < host.portMin || candidate > host.portMax) return false;
const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
@@ -281,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin
}
export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.');
if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
if (!instanceId) throw new Error('An immutable instance ID is required.');
src/lib/server/hosts/schema.ts
+7
View File
@@ -40,6 +40,13 @@ export function ensureHostSchema(handle: Database.Database): void {
PRIMARY KEY(host_id,bind_address,protocol,port)
);
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
CREATE TABLE IF NOT EXISTS ec_host_decommissions (
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued'
CHECK(stage IN ('queued','running','cleaned','failed','completed')),
requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT
);
`);
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');