import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import Database from 'better-sqlite3'; import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 }; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); }); async function enrolled() { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Game host' }); const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); return { ...hosts, host, credentials }; } async function finishCleanup(hosts: Awaited>) { const result = hosts.deleteHost(hosts.host.id); const operation = hosts.getHostOperation(result.operationId!)!; hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); const token = operation.payload.receiptToken as string; hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' }); hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' }); return operation; } async function instance(hostId: string, retired = false) { const { db } = await import('../../src/lib/server/db/index.js'); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at) VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)`) .run(hostId, new Date().toISOString(), retired ? new Date().toISOString() : null); } describe('host deletion', () => { it('adds the deletion marker to existing host schemas without losing hosts', async () => { const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js'); const legacy = new Database(':memory:'); try { legacy.exec("CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')"); ensureHostSchema(legacy); ensureHostSchema(legacy); expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({ id: 'old', name: 'Existing host', deleted_at: null }); } finally { legacy.close(); } }); it('deletes an unenrolled host and invalidates its unused enrollment', async () => { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.createHost({ name: 'Not installed' }); const enrollment = hosts.createEnrollment(host.id); expect(hosts.hostDeletionStatus(host.id)).toEqual({ allowed: true, blockers: [] }); hosts.deleteHost(host.id); expect(hosts.getHost(host.id)).toBeNull(); expect(hosts.listHosts()).toEqual([]); expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired'); expect(() => hosts.createEnrollment(host.id)).toThrow('not found'); expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 })); }); it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => { const hosts = await enrolled(); const other = hosts.createHost({ name: 'Keep' }); const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory); const enrollment = hosts.createEnrollment(hosts.host.id); const completed = hosts.queueHostOperation(hosts.host.id, 'host.inspect', {}); hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'completed', result: { hostname: 'games-1' } }); await instance(hosts.host.id, true); const file = join(fixture.root, 'game-files-sentinel'); await writeFile(file, 'Game files remain'); const { db } = await import('../../src/lib/server/db/index.js'); const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js'); ensureGameFileSchema(db()); db().prepare("INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)").run(new Date().toISOString()); db().prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume) VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id); db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString()); const queued = hosts.deleteHost(hosts.host.id); expect(queued.deleted).toBe(false); expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued'); const { nativeLaunchSelection } = await import('../../src/lib/server/provisioning/native.js'); expect(() => nativeLaunchSelection({ hostId: hosts.host.id, libraryId: 'version', image: 'plutainer:test' }, 't5')) .toThrow('The selected host is being removed'); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); await finishCleanup(hosts); expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull(); expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id); expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired'); expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]); expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } }); expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2); expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) }); expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) }); expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 }); expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({ count: 1 }); expect(db().prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'").get()).toEqual({ state: 'ready', source_host_id: hosts.host.id }); expect(await readFile(file, 'utf8')).toBe('Game files remain'); expect(() => hosts.heartbeat(hosts.host.id, inventory)).toThrow('not found'); expect(() => hosts.rotateCredential(hosts.host.id)).toThrow('not found'); expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('not found'); expect(() => hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' })).toThrow('not found'); expect(() => hosts.updateHost(hosts.host.id, { mode: 'active' })).toThrow('not found'); }); it('cancels only undelivered queued operations and retains their journal', async () => { const hosts = await enrolled(); const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map(kind => hosts.queueHostOperation(hosts.host.id, kind, {})); expect(hosts.hostDeletionStatus(hosts.host.id).allowed).toBe(true); hosts.deleteHost(hosts.host.id); const { db } = await import('../../src/lib/server/db/index.js'); for (const operation of operations) { expect(hosts.getHostOperation(operation.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('before this operation was delivered') }); expect(db().prepare('SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id').all(operation.id)).toEqual([{ state: 'pending' }, { state: 'failed' }]); } }); it('blocks live instances even when no server record or port reservation exists', async () => { const hosts = await enrolled(); await instance(hosts.host.id); const status = hosts.hostDeletionStatus(hosts.host.id); expect(status.allowed).toBe(false); expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1'); expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); }); it('blocks an accepted unplaced pinned launch until it is cancelled', async () => { const hosts = await enrolled(); const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js'); const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); const { db } = await import('../../src/lib/server/db/index.js'); const queued = queueNativeCreationIntent({ server: 't5_zombie_theater_1', game: 't5', mapSlug: 'theater', plutoKey: 'saved-key', selection: { hostId: hosts.host.id, libraryId: 'library', image: 'plutainer:test' } }); expect(queued.ok).toBe(true); if (!queued.ok) throw new Error(queued.message); expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: false, blockers: [expect.stringContaining('Cancel the queued server launches pinned to this host')] }); expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); expect(hosts.getHost(hosts.host.id)?.decommission).toBeNull(); expect(cancelCreation(queued.deploymentId!)).toBeTruthy(); expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] }); await finishCleanup(hosts); expect(hosts.getHost(hosts.host.id)).toBeNull(); }); it('does not pin a host for automatic launches or launches assigned elsewhere', async () => { const hosts = await enrolled(); const other = hosts.createHost({ name: 'Other host' }); const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js'); for (const [index, selection] of [ { hostId: hosts.host.id, automatic: true }, { hostId: other.id, automatic: false } ].entries()) { expect(queueNativeCreationIntent({ server: `t5_zombie_theater_${index + 1}`, game: 't5', mapSlug: 'theater', plutoKey: 'saved-key', selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } }).ok).toBe(true); } expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] }); expect(hosts.hostDeletionStatus(other.id).allowed).toBe(false); await finishCleanup(hosts); expect(hosts.getHost(hosts.host.id)).toBeNull(); }); it('blocks residual port reservations independently of live instances', async () => { const hosts = await enrolled(); hosts.reserveHostPort(hosts.host.id, 'unresolved-instance'); expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] }); expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); hosts.releaseHostPorts('unresolved-instance'); await finishCleanup(hosts); expect(hosts.getHost(hosts.host.id)).toBeNull(); }); it.each(['pending', 'accepted', 'running', 'unknown'] as const)('blocks unresolved %s work after delivery without revoking access or cancelling other work', async state => { const hosts = await enrolled(); const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {}); hosts.pendingHostOperations(hosts.host.id); if (state !== 'pending') hosts.reportHostOperation(hosts.host.id, operation.id, { state }); const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {}); hosts.updateHost(hosts.host.id, { mode: 'disabled' }); expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('delivered operation')] }); expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); expect(hosts.getHostOperation(queued.id)?.state).toBe('pending'); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }); hosts.deleteHost(hosts.host.id); expect(hosts.getHostOperation(queued.id)?.state).toBe('failed'); }); it('rechecks blockers on DELETE even after a successful eligibility check', async () => { const hosts = await enrolled(); const route = await import('../../src/routes/api/hosts/[id]/+server.js'); const url = `https://ec.test/api/hosts/${hosts.host.id}`; const params = { id: hosts.host.id }; expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({ deletion: { allowed: true, blockers: [] } }); const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {}); hosts.pendingHostOperations(hosts.host.id); const blocked = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }); expect(blocked.status).toBe(409); expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] }); hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' }); const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }); expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) }); await finishCleanup(hosts); expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404); }); it('keeps DELETE restricted to admin sessions and same-origin management requests', async () => { const hosts = await enrolled(); const { routePermission, roleAllows } = await import('../../src/lib/server/auth/access.js'); const permission = routePermission('/api/hosts/[id]', 'DELETE'); expect(roleAllows('admin', permission)).toBe(true); expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false); const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js'); for (const headers of [new Headers({ origin: 'https://attacker.test' }), new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })]) { expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 })); } expect(hosts.getHost(hosts.host.id)).not.toBeNull(); }); it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => { const hosts = await enrolled(); hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] }); expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] }); expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 })); expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy(); expect(hosts.getHost(hosts.host.id)?.mode).toBe('active'); }); it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => { const hosts = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id); const queued = hosts.deleteHost(hosts.host.id); expect(hosts.deleteHost(hosts.host.id)).toEqual(queued); expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } }); expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]); expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup'); expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup'); expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup'); expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false); expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup'); expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup'); hosts.heartbeat(hosts.host.id, inventory); expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]); }); it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => { const hosts = await enrolled(); const queued = hosts.deleteHost(hosts.host.id); const operation = hosts.getHostOperation(queued.operationId!)!; const token = operation.payload.receiptToken as string; const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage }); expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 })); hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 })); expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 })); expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker'); receipt('cleaned'); expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned'); expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id); const route = await import('../../src/routes/api/hosts/[id]/+server.js'); const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text(); expect(detail).not.toContain(token); receipt('completed'); expect(hosts.getHost(hosts.host.id)).toBeNull(); expect(receipt('completed')).toEqual({ acknowledged: true }); expect(receipt('cleaned')).toEqual({ acknowledged: true }); expect(hosts.getHostOperation(operation.id)?.state).toBe('completed'); expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull(); }); it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => { const hosts = await enrolled(); const first = hosts.deleteHost(hosts.host.id); hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' }); expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true }); expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy(); const second = hosts.deleteHost(hosts.host.id); expect(second.operationId).not.toBe(first.operationId); const operation = hosts.getHostOperation(second.operationId!)!; hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' }); expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' }); expect(hosts.deleteHost(hosts.host.id)).toEqual(second); await finishCleanup(hosts); expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed'); }); it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => { const hosts = await enrolled(); const queued = hosts.deleteHost(hosts.host.id); const operation = hosts.getHostOperation(queued.operationId!)!; hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' }); const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js'); const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage }) }), params: { id: operation.id } }); expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401); expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200); expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200); expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200); }); });