josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove host files and uninstall agent when deleting hosts

Browse Source
This commit is contained in:
josh
2026-10-04 10:20:51 -04:00
parent 62d7817ea3
commit 020894b816
20 changed files with 1743 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/auth-access.test.ts
+1
View File
@@ -92,6 +92,7 @@ describe('authentication and machine boundaries', () => {
it.each([
['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/agent/operations', 'GET'],
['/api/agent/operations/[id]', 'POST'], ['/api/agent/libraries/[id]/files/[...path]', 'GET'],
['/api/agent/decommission/[id]', 'POST'],
['/api/agent/imports/[id]/files/[...path]', 'PUT'], ['/api/agent/imports/[id]/complete', 'POST'],
['/api/agent/install.sh', 'GET'], ['/api/agent/release/[file]', 'GET'], ['/api/telemetry', 'POST']
])('preserves endpoint-specific machine authentication for %s', (route, method) => {
tests/unit/host-deletion.test.ts
+105 -6
View File
@@ -5,7 +5,7 @@ import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const inventory = { hostname: 'games-1', protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); });
@@ -16,6 +16,16 @@ async function enrolled() {
return { ...hosts, host, credentials };
}
async function finishCleanup(hosts: Awaited<ReturnType<typeof enrolled>>) {
const result = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(result.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const token = operation.payload.receiptToken as string;
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' });
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' });
return operation;
}
async function instance(hostId: string, retired = false) {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
@@ -46,7 +56,7 @@ describe('host deletion', () => {
expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 }));
});
it('deletes an enrolled empty host, revokes credentials, and preserves files and history', async () => {
it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => {
const hosts = await enrolled();
const other = hosts.createHost({ name: 'Keep' });
const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
@@ -64,14 +74,18 @@ describe('host deletion', () => {
VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id);
db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString());
hosts.deleteHost(hosts.host.id);
const queued = hosts.deleteHost(hosts.host.id);
expect(queued.deleted).toBe(false);
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
await finishCleanup(hosts);
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id);
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]);
expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } });
expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(1);
expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2);
expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) });
expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) });
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 });
@@ -112,7 +126,7 @@ describe('host deletion', () => {
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
hosts.releaseHostPorts('unresolved-instance');
hosts.deleteHost(hosts.host.id);
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
@@ -145,7 +159,8 @@ describe('host deletion', () => {
expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] });
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
expect(removed.status).toBe(200); expect(await removed.json()).toEqual({ deleted: true });
expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) });
await finishCleanup(hosts);
expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404);
});
@@ -161,4 +176,88 @@ describe('host deletion', () => {
}
expect(hosts.getHost(hosts.host.id)).not.toBeNull();
});
it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => {
const hosts = await enrolled();
hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] });
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
expect(hosts.getHost(hosts.host.id)?.mode).toBe('active');
});
it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => {
const hosts = await enrolled();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id);
const queued = hosts.deleteHost(hosts.host.id);
expect(hosts.deleteHost(hosts.host.id)).toEqual(queued);
expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } });
expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]);
expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup');
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup');
expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup');
expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false);
expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup');
expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup');
hosts.heartbeat(hosts.host.id, inventory);
expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]);
});
it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
const token = operation.payload.receiptToken as string;
const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage });
expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 }));
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 }));
expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 }));
expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker');
receipt('cleaned');
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text();
expect(detail).not.toContain(token);
receipt('completed');
expect(hosts.getHost(hosts.host.id)).toBeNull();
expect(receipt('completed')).toEqual({ acknowledged: true });
expect(receipt('cleaned')).toEqual({ acknowledged: true });
expect(hosts.getHostOperation(operation.id)?.state).toBe('completed');
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
});
it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => {
const hosts = await enrolled();
const first = hosts.deleteHost(hosts.host.id);
hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' });
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true });
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
const second = hosts.deleteHost(hosts.host.id);
expect(second.operationId).not.toBe(first.operationId);
const operation = hosts.getHostOperation(second.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' });
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' });
expect(hosts.deleteHost(hosts.host.id)).toEqual(second);
await finishCleanup(hosts);
expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed');
});
it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js');
const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, {
method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage })
}), params: { id: operation.id } });
expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401);
expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
});
});