Remove host files and uninstall agent when deleting hosts

src/lib/hosts.ts

@@ -1,13 +1,27 @@
 export type HostMode = 'active' | 'draining' | 'disabled';
-export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled';
+export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
 export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
 
 export function hostReadinessLabel(readiness: HostReadiness): string {
     return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
-        ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled' }[readiness];
+        ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
 }
 
 export interface HostDeletion { allowed: boolean; blockers: string[] }
+export interface HostDecommission {
+    operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
+    requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
+}
+export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
+    if (!host.decommission) return '';
+    switch (host.decommission.stage) {
+        case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
+        case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
+        case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
+        case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
+        case 'completed': return 'Host files and agent removal confirmed.';
+    }
+}
 
 export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
 export interface HostInventory {
@@ -16,6 +30,7 @@ export interface HostInventory {
     storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
     directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
     currentOperation?: string | null; [key: string]: unknown;
+    capabilities?: string[];
 }
 export interface Host {
     id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
@@ -23,6 +38,7 @@ export interface Host {
     lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
     readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
     enrolled: boolean; credentialActive: boolean;
+    decommission: HostDecommission | null;
 }
 export interface HostOperation {
     id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;

src/lib/server/hosts/index.ts

@@ -1,8 +1,8 @@
-import { createHash, randomBytes, randomUUID } from 'node:crypto';
+import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
-import type { Host, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
+import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex
 function database() { const handle = db(); ensureHostSchema(handle); return handle; }
 type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
 const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
+type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null };
+function decommissionRow(hostId: string): DecommissionRow | undefined {
+    return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined;
+}
+function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
+    return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
+        error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
+}
 function hostFromRow(row: HostRow): Host {
     const inventory: HostInventory = JSON.parse(row.inventory);
     const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
@@ -29,11 +37,13 @@ function hostFromRow(row: HostRow): Host {
     if (!libraryReady) reasons.push('Download and verify a game-file library.');
     if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
     if (row.mode === 'disabled') reasons.push('Host is disabled.');
+    const decommission = presentDecommission(decommissionRow(row.id));
+    if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
     return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
         bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
         createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
-        readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
-        readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
+        readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
+        readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
 }
 export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
 export function getHost(id: string): Host | null {
@@ -43,14 +53,18 @@ export function getHost(id: string): Host | null {
 function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
 
 export class HostDeletionError extends Error {
-    constructor(message: string, readonly status: 404 | 409, readonly blockers: string[] = []) { super(message); }
+    constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); }
 }
 
 /** Controller records can be removed only after all remote instance ownership is resolved. */
 export function hostDeletionStatus(id: string): HostDeletion {
-    if (!getHost(id)) throw new HostDeletionError('Host not found.', 404);
+    const host = getHost(id);
+    if (!host) throw new HostDeletionError('Host not found.', 404);
+    if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
     const handle = database();
     const blockers: string[] = [];
+    if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.');
+    if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.');
     const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[];
     if (instances.length) {
         const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
@@ -70,20 +84,68 @@ export function hostDeletionStatus(id: string): HostDeletion {
     return { allowed: blockers.length === 0, blockers };
 }
 
-/** Retain references and audit history; deletion never sends remote cleanup commands. */
-export function deleteHost(id: string): void {
+/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
+export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
     const handle = database();
-    handle.transaction(() => {
+    return handle.transaction(() => {
+        const host = getHost(id);
+        if (!host) throw new HostDeletionError('Host not found.', 404);
+        if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId };
         const status = hostDeletionStatus(id);
         if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
         const timestamp = now();
-        const reason = 'Host deleted before this operation was delivered to the agent.';
+        const reason = 'Host deletion requested before this operation was delivered to the agent.';
         handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
             SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id);
         handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id);
-        handle.prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, id);
         handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id);
-        handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
+        if (!host.enrolled) {
+            handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
+            return { deleted: true };
+        }
+        const operationId = randomUUID();
+        const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
+        const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
+        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
+        handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
+        handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
+            ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
+            .run(id, operationId, digest(token), timestamp, timestamp);
+        handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
+        return { deleted: false, operationId };
+    }).immediate();
+}
+
+export function receiveDecommissionReceipt(operationId: string, token: string, input: Record<string, unknown>): { acknowledged: true } {
+    if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
+    return database().transaction(() => {
+        const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined;
+        if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
+        if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
+        if (row.stage === 'completed') return { acknowledged: true as const };
+        const operation = getHostOperation(operationId)!;
+        if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
+        const timestamp = now();
+        if (input.stage === 'failed') {
+            database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?")
+                .run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id);
+            return { acknowledged: true as const };
+        }
+        if (input.stage === 'cleaned') {
+            database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id);
+            return { acknowledged: true as const };
+        }
+        if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409);
+        database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
+        database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId);
+        database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp);
+        database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id);
+        database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id);
+        if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) {
+            database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
+        }
+        database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id);
+        return { acknowledged: true as const };
     }).immediate();
 }
 function textField(value: unknown, label: string, max = 200): string {
@@ -102,6 +164,7 @@ export function createHost(input: Record<string, unknown>): Host {
 }
 export function updateHost(id: string, input: Record<string, unknown>): Host {
     const host = requireHost(id);
+    if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
     const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
     const mode = input.mode ?? host.mode;
     if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
@@ -117,7 +180,8 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
 }
 
 export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
-    requireHost(hostId);
+    const host = requireHost(hostId);
+    if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.');
     if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
     const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
     const expiresAt = new Date(Date.now() + ttlMs).toISOString();
@@ -143,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory {
     }
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
+    if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
     if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
         !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
     return inventory;
@@ -169,7 +234,8 @@ export function authenticateCredential(token: string): Host | null {
     return getHost(row.host_id);
 }
 export function revokeCredentials(hostId: string): void {
-    requireHost(hostId);
+    const host = requireHost(hostId);
+    if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
     database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
 }
 export function rotateCredential(hostId: string, nextCredential?: string): string {
@@ -207,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row =
 export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
 export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
     const host = requireHost(hostId);
+    if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
+    if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
     if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
     if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
     if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
@@ -245,6 +313,7 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         requireHost(hostId);
         const operation = getHostOperation(id);
         if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
+        if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
         if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
         if (['completed','failed'].includes(operation.state)) return operation;
         const state = input.state as HostOperationState;
@@ -253,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         const result = input.result === undefined ? null : JSON.stringify(input.result);
         if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
         database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
+        if (operation.kind === 'host.decommission') {
+            if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
+            else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
+        }
         if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
         return getHostOperation(id)!;
     })();
@@ -270,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
 
 export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
     const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
+    if (host.decommission) return false;
     if (candidate < host.portMin || candidate > host.portMax) return false;
     const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
     return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
@@ -281,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin
 }
 export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
     const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
+    if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.');
     if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
     if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
     if (!instanceId) throw new Error('An immutable instance ID is required.');

src/lib/server/hosts/schema.ts

@@ -40,6 +40,13 @@ export function ensureHostSchema(handle: Database.Database): void {
             PRIMARY KEY(host_id,bind_address,protocol,port)
         );
         CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_decommissions (
+            host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
+            operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
+            token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued'
+                CHECK(stage IN ('queued','running','cleaned','failed','completed')),
+            requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT
+        );
     `);
     const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
     if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');

src/lib/ui/components/DeleteHostDialog.svelte

@@ -4,7 +4,7 @@
     import Dialog from './primitives/Dialog.svelte';
 
     let { open = $bindable(false), host, ondeleted }:
-        { open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: () => void | Promise<void> } = $props();
+        { open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise<void> } = $props();
     let deletion = $state<HostDeletion | null>(null);
     let checking = $state(false); let deleting = $state(false); let error = $state('');
 
@@ -36,19 +36,20 @@
                 throw new Error(body.error ?? body.message ?? 'Could not delete this host.');
             }
             open = false;
-            await ondeleted();
+            await ondeleted(body);
         } catch (cause) { error = (cause as Error).message; }
         finally { deleting = false; }
     }
 </script>
 
 <Dialog bind:open title={`Delete ${host?.name ?? 'host'}?`} busy={deleting}
-    description="Remove this host from Electric Cherry and revoke its agent access and unused installer links.">
+    description="Permanently remove Electric Cherry's files from this host, including its downloaded game files, then uninstall the agent.">
     <div class="content">
-        <p>The agent and game files stay on the machine. Published game libraries and operation history are kept. Commands that have never been sent to the host will be cancelled.</p>
+        <p>This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.</p>
+        <p>The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.</p>
         {#if checking}<p role="status">Checking servers and unfinished operations…</p>
         {:else if deletion && !deletion.allowed}<div role="status" class="blocked"><strong>This host still has work to finish:</strong><ul>{#each deletion.blockers as blocker}<li>{blocker}</li>{/each}</ul><p>Open <a href={`/hosts/${host?.id}`} onclick={() => open = false}>host details</a> to review its operations or drain its servers.</p></div>
-        {:else if deletion?.allowed}<p>This host is ready to delete. Adding it again requires a new enrollment.</p>{/if}
+        {:else if deletion?.allowed}<p>Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.</p>{/if}
         {#if error}<p role="alert" class="ui-error">{error}</p>{/if}
     </div>
     {#snippet actions()}

src/routes/api/agent/decommission/[id]/+server.ts

@@ -0,0 +1,14 @@
+import { json } from '@sveltejs/kit';
+import { HostDeletionError, receiveDecommissionReceipt } from '$lib/server/hosts/index.js';
+import { objectBody, requireAgentTransport } from '$lib/server/hosts/http.js';
+
+/** The uninstall worker retains only this narrowly scoped credential after removing the agent. */
+export async function POST({ request, params }: { request: Request; params: { id: string } }) {
+    requireAgentTransport(request);
+    const token = request.headers.get('authorization')?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1] ?? '';
+    try { return json(receiveDecommissionReceipt(params.id, token, await objectBody(request, 8192)), { headers: { 'Cache-Control': 'no-store' } }); }
+    catch (cause) {
+        if (cause instanceof HostDeletionError) return json({ error: cause.message }, { status: cause.status });
+        throw cause;
+    }
+}

src/routes/api/hosts/[id]/+server.ts

@@ -15,7 +15,7 @@ export async function PATCH({ request, params }: Event) {
 
 export function DELETE({ request, params }: Event) {
     requireManagementOrigin(request);
-    try { deleteHost(params.id); return json({ deleted: true }); }
+    try { const result = deleteHost(params.id); return json(result, { status: result.deleted ? 200 : 202 }); }
     catch (cause) {
         if (cause instanceof HostDeletionError) return json({ error: cause.message, blockers: cause.blockers }, { status: cause.status });
         return hostApiError(cause);

src/routes/hosts/+page.svelte

@@ -2,7 +2,7 @@
     import { onMount } from 'svelte';
     import { page } from '$app/stores';
     import type { Host } from '$lib/hosts.js';
-    import { hostReadinessLabel } from '$lib/hosts.js';
+    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -18,8 +18,8 @@
         try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
         catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
     }
-    async function deleted() {
-        notice = `${deletingHost?.name ?? 'Host'} deleted.`;
+    async function deleted(result: { deleted: boolean }) {
+        notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
         if (enrollment?.hostId === deletingHost?.id) enrollment = null;
         deletingHost = null;
         await load();
@@ -51,7 +51,7 @@
         {#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
     </section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
     {#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
-        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>Manage host</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>Delete host</Button>{/if}</div></article>{/each}</div>
+        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
     {/if}
 </section>
 <DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />

src/routes/hosts/[id]/+page.svelte

@@ -3,7 +3,7 @@
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
     import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
-    import { hostReadinessLabel } from '$lib/hosts.js';
+    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -14,10 +14,11 @@
     let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null);
     const endpoint = $derived(`/api/hosts/${$page.params.id}`);
     const canManage = $derived($page.data.user?.role === 'admin');
+    const cleaning = $derived(!!host?.decommission);
     function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
     async function load(initialize = false) {
         try {
-            const response = await fetch(endpoint); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
+            const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
             if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; }
         } catch (cause) { error = (cause as Error).message; } finally { loading = false; }
     }
@@ -33,30 +34,31 @@
 
 <svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>
 <section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history.">
-    {#snippet actions()}{#if canManage && host}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>Delete host</Button>{/if}{/snippet}
+    {#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>{cleaning ? 'Retry cleanup' : 'Delete host'}</Button>{/if}{/snippet}
 </SectionHeader>
     {#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if}
     {#if loading}<p role="status">Loading host…</p>{:else if host}
+        {#if host.decommission}<section class="panel" aria-live="polite"><h2>Host cleanup</h2><p>{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}<p>Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.</p>{#if host.decommission.stage === 'cleaned'}<p>The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.</p>{/if}</section>{/if}
         <section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
             {#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if}
             {#if host.readiness === 'needs-game-files'}<p><a href="/game-files">Choose a published game-file version and download it to this host →</a></p>{/if}
-            {#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
+            {#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
             <p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
         </section>
         <div class="grid"><section class="panel"><h2>Inventory</h2><dl>
             <dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
-        </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy}>
+        </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
             <label>Friendly name<input required maxlength="200" bind:value={name} /></label><div class="fields"><label>First game port<input type="number" min="1" max="65535" required bind:value={portMin} /></label><label>Last game port<input type="number" min={portMin} max="65535" required bind:value={portMax} /></label></div><label>Local bind address<input required bind:value={bindAddress} /></label><label>Public / NAT address<input bind:value={publicAddress} placeholder="Optional public hostname or address" /></label><label>Minimum free game storage (GiB)<input type="number" min="0" step="0.1" required bind:value={minimumFreeGB} /></label><Button type="submit" disabled={busy}>Save settings</Button>
         </fieldset></form>{:else}<dl><dt>Friendly name</dt><dd>{host.name}</dd><dt>Game ports</dt><dd>{host.portMin}–{host.portMax}</dd><dt>Local bind address</dt><dd>{host.bindAddress}</dd><dt>Public / NAT address</dt><dd>{host.publicAddress || 'Not configured'}</dd><dt>Minimum free storage</dt><dd>{bytes(host.minimumFreeBytes)}</dd></dl><p>Host configuration is managed by administrators.</p>{/if}<p>Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.</p></section></div>
         <div class="grid"><section class="panel"><h2>Game files and runtime</h2><p><a href="/game-files">Manage game-file libraries →</a></p>{#if host.inventory.libraries?.length}<ul>{#each host.inventory.libraries as library}<li>{library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}</li>{/each}</ul>{:else}<p>No cached game libraries reported.</p>{/if}
-            {#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
-        </section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
+            {#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy || cleaning}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
+        </section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy || (cleaning && !host.decommission?.retryable)} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || cleaning || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || cleaning || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
             {#if enrollment}<p>Expires {new Date(enrollment.expiresAt).toLocaleString()}</p><pre>{enrollment.command}</pre>{/if}{:else}<p>Administrators manage enrollment and agent credentials.</p>{/if}
         </section></div>
         <section class="panel"><h2>Operation history</h2>{#if !operations.length}<p>No remote operations yet.</p>{:else}<div class="table-wrap"><table><thead><tr><th>Operation</th><th>State</th><th>Updated</th><th>Result</th></tr></thead><tbody>{#each operations as operation}<tr><td>{operation.kind}<small>{operation.id}</small></td><td>{operation.state}</td><td>{new Date(operation.updatedAt).toLocaleString()}</td><td>{operation.error ?? '—'}</td></tr>{/each}</tbody></table></div>{/if}</section>
     {/if}
 </section>
-<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={() => goto('/hosts')} />
+<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={async (result) => { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />
 
 <style>
     .page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; min-width:0; } h2 { margin:0 0 16px; font-size:var(--text-md); text-transform:capitalize; } p, li, dd, dt, span { font-size:var(--text-sm); line-height:1.65; } p,li,dt,span { color:var(--text-secondary); } .grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:24px; } .row,.actions,.fields { display:flex; gap:12px; flex-wrap:wrap; } .row { justify-content:space-between; } .actions { margin-top:20px; } dl { display:grid; grid-template-columns:1fr 1.3fr; gap:12px; margin:0; } dd { margin:0; overflow-wrap:anywhere; } fieldset { display:grid; gap:16px; border:0; padding:0; margin:0; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; min-width:0; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); } .fields>* { flex:1; min-width:120px; } pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-xs); } .table-wrap { overflow:auto; } table { width:100%; border-collapse:collapse; text-align:left; font-size:var(--text-sm); } td,th { padding:12px; border-bottom:1px solid var(--border-subtle); } small { display:block; color:var(--text-secondary); font-size:var(--text-xs); margin-top:4px; } @media(max-width:900px) { .grid { grid-template-columns:1fr; } }