Files
josh 8d1bb69591
CI / checks (push) Successful in 5m6s
Use ordered SQLite migrations (Refs #75)
2026-10-10 00:01:18 -04:00

498 lines
24 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import Database from 'better-sqlite3';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const inventory = {
hostname: 'games-1',
protocolVersion: 1,
capabilities: ['host.decommission'],
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3
};
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
});
afterEach(async () => {
vi.unstubAllEnvs();
await fixture.cleanup();
});
async function enrolled() {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Game host' });
const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
return { ...hosts, host, credentials };
}
async function finishCleanup(hosts: Awaited<ReturnType<typeof enrolled>>) {
const result = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(result.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const token = operation.payload.receiptToken as string;
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' });
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' });
return operation;
}
async function instance(hostId: string, retired = false) {
const { db } = await import('../../src/lib/server/db/index.js');
db()
.prepare(
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)`
)
.run(hostId, new Date().toISOString(), retired ? new Date().toISOString() : null);
}
describe('host deletion', () => {
it('adds the deletion marker to existing host schemas without losing hosts', async () => {
const { migrateHostSchema } = await import('../../src/lib/server/hosts/schema.js');
const legacy = new Database(':memory:');
try {
legacy.exec(
"CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')"
);
migrateHostSchema(legacy);
migrateHostSchema(legacy);
expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({
id: 'old',
name: 'Existing host',
deleted_at: null
});
} finally {
legacy.close();
}
});
it('deletes an unenrolled host and invalidates its unused enrollment', async () => {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Not installed' });
const enrollment = hosts.createEnrollment(host.id);
expect(hosts.hostDeletionStatus(host.id)).toEqual({ allowed: true, blockers: [] });
hosts.deleteHost(host.id);
expect(hosts.getHost(host.id)).toBeNull();
expect(hosts.listHosts()).toEqual([]);
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
expect(() => hosts.createEnrollment(host.id)).toThrow('not found');
expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 }));
});
it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => {
const hosts = await enrolled();
const other = hosts.createHost({ name: 'Keep' });
const otherCredentials = hosts.enrollHost(
hosts.createEnrollment(other.id).token,
inventory
);
const enrollment = hosts.createEnrollment(hosts.host.id);
const completed = hosts.queueHostOperation(hosts.host.id, 'host.inspect', {});
hosts.reportHostOperation(hosts.host.id, completed.id, {
state: 'completed',
result: { hostname: 'games-1' }
});
await instance(hosts.host.id, true);
const file = join(fixture.root, 'game-files-sentinel');
await writeFile(file, 'Game files remain');
const { db } = await import('../../src/lib/server/db/index.js');
db()
.prepare(
"INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)"
)
.run(new Date().toISOString());
db()
.prepare(
`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`
)
.run(new Date().toISOString(), file, hosts.host.id);
db()
.prepare(
"INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)"
)
.run(hosts.host.id, new Date().toISOString());
const queued = hosts.deleteHost(hosts.host.id);
expect(queued.deleted).toBe(false);
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
const { nativeLaunchSelection } =
await import('../../src/lib/server/provisioning/native.js');
expect(() =>
nativeLaunchSelection(
{ hostId: hosts.host.id, libraryId: 'version', image: 'plutainer:test' },
't5'
)
).toThrow('The selected host is being removed');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
await finishCleanup(hosts);
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id);
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
expect(hosts.listHosts().map((host) => host.id)).toEqual([other.id]);
expect(hosts.getHostOperation(completed.id)).toMatchObject({
state: 'completed',
result: { hostname: 'games-1' }
});
expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2);
expect(
db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)
).toEqual({ mode: 'disabled', deleted_at: expect.any(String) });
expect(
db()
.prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?')
.get(hosts.host.id)
).toEqual({ revoked_at: expect.any(String) });
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({
count: 1
});
expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({
count: 1
});
expect(
db()
.prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'")
.get()
).toEqual({ state: 'ready', source_host_id: hosts.host.id });
expect(await readFile(file, 'utf8')).toBe('Game files remain');
expect(() => hosts.heartbeat(hosts.host.id, inventory)).toThrow('not found');
expect(() => hosts.rotateCredential(hosts.host.id)).toThrow('not found');
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow(
'not found'
);
expect(() =>
hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' })
).toThrow('not found');
expect(() => hosts.updateHost(hosts.host.id, { mode: 'active' })).toThrow('not found');
});
it('cancels only undelivered queued operations and retains their journal', async () => {
const hosts = await enrolled();
const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map((kind) =>
hosts.queueHostOperation(hosts.host.id, kind, {})
);
expect(hosts.hostDeletionStatus(hosts.host.id).allowed).toBe(true);
hosts.deleteHost(hosts.host.id);
const { db } = await import('../../src/lib/server/db/index.js');
for (const operation of operations) {
expect(hosts.getHostOperation(operation.id)).toMatchObject({
state: 'failed',
error: expect.stringContaining('before this operation was delivered')
});
expect(
db()
.prepare(
'SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id'
)
.all(operation.id)
).toEqual([{ state: 'pending' }, { state: 'failed' }]);
}
});
it('blocks live instances even when no server record or port reservation exists', async () => {
const hosts = await enrolled();
await instance(hosts.host.id);
const status = hosts.hostDeletionStatus(hosts.host.id);
expect(status.allowed).toBe(false);
expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1');
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(
expect.objectContaining({ status: 409 })
);
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
});
it('blocks an accepted unplaced pinned launch until it is cancelled', async () => {
const hosts = await enrolled();
const { queueNativeCreationIntent } =
await import('../../src/lib/server/provisioning/native.js');
const { cancelCreation } =
await import('../../src/lib/server/provisioning/cancel-creation.js');
const { db } = await import('../../src/lib/server/db/index.js');
const queued = queueNativeCreationIntent({
server: 't5_zombie_theater_1',
game: 't5',
mapSlug: 'theater',
plutoKey: 'saved-key',
selection: { hostId: hosts.host.id, libraryId: 'library', image: 'plutainer:test' }
});
expect(queued.ok).toBe(true);
if (!queued.ok) throw new Error(queued.message);
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({
allowed: false,
blockers: [
expect.stringContaining('Cancel the queued server launches pinned to this host')
]
});
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(
expect.objectContaining({ status: 409 })
);
expect(hosts.getHost(hosts.host.id)?.decommission).toBeNull();
expect(cancelCreation(queued.deploymentId!)).toBeTruthy();
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] });
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it('does not pin a host for automatic launches or launches assigned elsewhere', async () => {
const hosts = await enrolled();
const other = hosts.createHost({ name: 'Other host' });
const { queueNativeCreationIntent } =
await import('../../src/lib/server/provisioning/native.js');
for (const [index, selection] of [
{ hostId: hosts.host.id, automatic: true },
{ hostId: other.id, automatic: false }
].entries()) {
expect(
queueNativeCreationIntent({
server: `t5_zombie_theater_${index + 1}`,
game: 't5',
mapSlug: 'theater',
plutoKey: `saved-key-${index}`,
selection: { ...selection, libraryId: 'library', image: 'plutainer:test' }
}).ok
).toBe(true);
}
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] });
expect(hosts.hostDeletionStatus(other.id).allowed).toBe(false);
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it('blocks residual port reservations independently of live instances', async () => {
const hosts = await enrolled();
hosts.reserveHostPort(hosts.host.id, 'unresolved-instance');
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({
allowed: false,
blockers: [expect.stringContaining('reserved game port')]
});
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(
expect.objectContaining({ status: 409 })
);
hosts.releaseHostPorts('unresolved-instance');
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it.each(['pending', 'accepted', 'running', 'unknown'] as const)(
'blocks unresolved %s work after delivery without revoking access or cancelling other work',
async (state) => {
const hosts = await enrolled();
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
hosts.pendingHostOperations(hosts.host.id);
if (state !== 'pending')
hosts.reportHostOperation(hosts.host.id, operation.id, { state });
const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {});
hosts.updateHost(hosts.host.id, { mode: 'disabled' });
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({
allowed: false,
blockers: [expect.stringContaining('delivered operation')]
});
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(
expect.objectContaining({ status: 409 })
);
expect(hosts.getHostOperation(queued.id)?.state).toBe('pending');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(
hosts.host.id
);
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
hosts.deleteHost(hosts.host.id);
expect(hosts.getHostOperation(queued.id)?.state).toBe('failed');
}
);
it('rechecks blockers on DELETE even after a successful eligibility check', async () => {
const hosts = await enrolled();
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
const url = `https://ec.test/api/hosts/${hosts.host.id}`;
const params = { id: hosts.host.id };
expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({
deletion: { allowed: true, blockers: [] }
});
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
hosts.pendingHostOperations(hosts.host.id);
const blocked = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
expect(blocked.status).toBe(409);
expect(await blocked.json()).toMatchObject({
error: expect.any(String),
blockers: [expect.stringContaining('delivered operation')]
});
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
expect(removed.status).toBe(202);
expect(await removed.json()).toMatchObject({
deleted: false,
operationId: expect.any(String)
});
await finishCleanup(hosts);
expect(
route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status
).toBe(404);
});
it('keeps DELETE restricted to admin sessions and same-origin management requests', async () => {
const hosts = await enrolled();
const { routePermission, roleAllows } = await import('../../src/lib/server/auth/access.js');
const permission = routePermission('/api/hosts/[id]', 'DELETE');
expect(roleAllows('admin', permission)).toBe(true);
expect(roleAllows('operator', permission)).toBe(false);
expect(roleAllows('viewer', permission)).toBe(false);
const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js');
for (const headers of [
new Headers({ origin: 'https://attacker.test' }),
new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })
]) {
expect(() =>
DELETE({
request: new Request('https://ec.test/api/hosts/id', {
method: 'DELETE',
headers
}),
params: { id: hosts.host.id }
})
).toThrow(expect.objectContaining({ status: 403 }));
}
expect(hosts.getHost(hosts.host.id)).not.toBeNull();
});
it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => {
const hosts = await enrolled();
hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] });
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({
allowed: false,
blockers: [expect.stringContaining('Update this host agent')]
});
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(
expect.objectContaining({ status: 409 })
);
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
expect(hosts.getHost(hosts.host.id)?.mode).toBe('active');
});
it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => {
const hosts = await enrolled();
const { db } = await import('../../src/lib/server/db/index.js');
db()
.prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?')
.run('2020-01-01T00:00:00Z', hosts.host.id);
const queued = hosts.deleteHost(hosts.host.id);
expect(hosts.deleteHost(hosts.host.id)).toEqual(queued);
expect(hosts.getHost(hosts.host.id)).toMatchObject({
connected: false,
readiness: 'deleting',
decommission: { stage: 'queued' }
});
expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]);
expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup');
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow(
'cleanup'
);
expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup');
expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false);
expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup');
expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup');
hosts.heartbeat(hosts.host.id, inventory);
expect(hosts.pendingHostOperations(hosts.host.id).map((operation) => operation.id)).toEqual(
[queued.operationId]
);
});
it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
const token = operation.payload.receiptToken as string;
const receipt = (stage: string, credential = token) =>
hosts.receiveDecommissionReceipt(operation.id, credential, { stage });
expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 }));
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(
expect.objectContaining({ status: 401 })
);
expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 }));
expect(() =>
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })
).toThrow('cleanup worker');
receipt('cleaned');
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
const detail = await route
.GET({
request: new Request('https://ec.test/api/hosts/' + hosts.host.id),
params: { id: hosts.host.id }
})
.text();
expect(detail).not.toContain(token);
receipt('completed');
expect(hosts.getHost(hosts.host.id)).toBeNull();
expect(receipt('completed')).toEqual({ acknowledged: true });
expect(receipt('cleaned')).toEqual({ acknowledged: true });
expect(hosts.getHostOperation(operation.id)?.state).toBe('completed');
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
});
it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => {
const hosts = await enrolled();
const first = hosts.deleteHost(hosts.host.id);
hosts.reportHostOperation(hosts.host.id, first.operationId!, {
state: 'failed',
error: 'Ownership marker requires repair'
});
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({
stage: 'failed',
retryable: true
});
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
const second = hosts.deleteHost(hosts.host.id);
expect(second.operationId).not.toBe(first.operationId);
const operation = hosts.getHostOperation(second.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, {
stage: 'failed',
error: 'Filesystem cleanup failed; retrying'
});
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({
stage: 'failed',
retryable: false,
error: 'Filesystem cleanup failed; retrying'
});
expect(hosts.deleteHost(hosts.host.id)).toEqual(second);
await finishCleanup(hosts);
expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed');
});
it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js');
const send = (stage: string, token: string) =>
POST({
request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({ stage })
}),
params: { id: operation.id }
});
expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401);
expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(
200
);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(
200
);
});
});