Add native host management and role-based operator authentication
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { mkdir, rename, stat } from 'node:fs/promises';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
|
||||
import { ensureGameFileSchema } from './schema.js';
|
||||
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
|
||||
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
|
||||
|
||||
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
|
||||
export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
|
||||
const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
|
||||
type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
|
||||
type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null };
|
||||
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id });
|
||||
function versionFromRow(row: VersionRow): GameLibraryVersion {
|
||||
const files = JSON.parse(row.manifest) as GameFileEntry[];
|
||||
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
|
||||
description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes,
|
||||
files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
|
||||
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id,
|
||||
sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id,
|
||||
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
|
||||
}
|
||||
export function getLibraryVersion(id: string): GameLibraryVersion | null {
|
||||
const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
|
||||
return row ? versionFromRow(row) : null;
|
||||
}
|
||||
export function getLibraryFile(versionId: string, path: string) {
|
||||
validateGamePath(path);
|
||||
return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation
|
||||
FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`)
|
||||
.get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null;
|
||||
}
|
||||
export function newestReadyLibrary(game: string): GameLibraryVersion | null {
|
||||
const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
|
||||
return row ? versionFromRow(row) : null;
|
||||
}
|
||||
export function listLibraries(): GameLibrary[] {
|
||||
reconcileLibraryOperations();
|
||||
return (database().prepare('SELECT * FROM game_file_libraries ORDER BY name').all() as Array<{ id: string; name: string; game: GameLibraryGame; description: string; created_at: string }>).map(row => ({
|
||||
id: row.id, name: row.name, game: row.game, description: row.description, createdAt: row.created_at,
|
||||
versions: (database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.id=? ORDER BY v.created_at DESC, v.rowid DESC').all(row.id) as VersionRow[]).map(versionFromRow)
|
||||
}));
|
||||
}
|
||||
export function getHostLibraryCache(hostId: string, versionId: string): HostLibraryCache | null {
|
||||
const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined;
|
||||
return row ? cacheFromRow(row) : null;
|
||||
}
|
||||
export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
|
||||
const version = getLibraryVersion(versionId);
|
||||
if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.');
|
||||
const cache = getHostLibraryCache(hostId, versionId);
|
||||
if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
|
||||
return version;
|
||||
}
|
||||
|
||||
function textField(input: unknown, label: string, max: number, required = true): string {
|
||||
if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`);
|
||||
return input.trim();
|
||||
}
|
||||
export function createLibraryImport(input: Record<string, unknown>): GameLibraryVersion {
|
||||
const game = input.game;
|
||||
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
|
||||
const hostId = textField(input.hostId, 'source host', 128);
|
||||
const host = getHost(hostId);
|
||||
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled source host.');
|
||||
const name = textField(input.name, 'library name', 160);
|
||||
const description = textField(input.description ?? '', 'description', 4000, false);
|
||||
const version = textField(input.version, 'version label', 120);
|
||||
const dlcInput = input.dlc ?? [];
|
||||
if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.');
|
||||
const dlc = dlcInput.map(value => textField(value, 'DLC name', 120));
|
||||
const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
|
||||
const handle = database();
|
||||
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
|
||||
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
|
||||
const id = randomUUID(); const now = new Date().toISOString();
|
||||
handle.transaction(() => {
|
||||
if (!library) {
|
||||
library = { id: randomUUID(), game };
|
||||
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
|
||||
}
|
||||
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
|
||||
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
|
||||
})();
|
||||
try {
|
||||
const operation = queueHostOperation(hostId, 'library.import', {
|
||||
importId: id, game, sourceVolume, manifestUrl: `/api/agent/imports/${id}`,
|
||||
uploadBaseUrl: `/api/agent/imports/${id}/files`, completeUrl: `/api/agent/imports/${id}/complete`
|
||||
}, { idempotencyKey: `library-import:${id}` });
|
||||
handle.prepare('UPDATE game_file_versions SET operation_id=? WHERE id=?').run(operation.id, id);
|
||||
} catch (cause) {
|
||||
handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=?").run((cause as Error).message, id);
|
||||
throw cause;
|
||||
}
|
||||
return getLibraryVersion(id)!;
|
||||
}
|
||||
|
||||
export function requireImportOwner(hostId: string, id: string): GameLibraryVersion {
|
||||
const version = getLibraryVersion(id);
|
||||
if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.');
|
||||
return version;
|
||||
}
|
||||
export function setImportManifest(hostId: string, id: string, input: unknown): GameLibraryVersion {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
const files = validateManifest(input); const checksum = manifestChecksum(files);
|
||||
if (version.manifestSha256) {
|
||||
if (version.manifestSha256 !== checksum) throw new Error('An import manifest cannot change. Start a new version.');
|
||||
return version;
|
||||
}
|
||||
if (version.state !== 'importing') throw new Error('This import is not writable.');
|
||||
database().transaction(() => {
|
||||
database().prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=?,size_bytes=? WHERE id=? AND state=\'importing\'').run(JSON.stringify(files), checksum, files.reduce((sum, file) => sum + file.size, 0), id);
|
||||
const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)');
|
||||
for (const file of files) insert.run(id, file.path, file.size, file.sha256);
|
||||
})();
|
||||
return getLibraryVersion(id)!;
|
||||
}
|
||||
export async function uploadImportFile(hostId: string, id: string, path: string, body: ReadableStream<Uint8Array> | null): Promise<void> {
|
||||
const file = getLibraryFile(id, path);
|
||||
if (!file || file.sourceHostId !== hostId) throw new Error('File is not listed in a manifest belonging to this host.');
|
||||
if (file.state !== 'importing' || publishing.has(id)) throw new Error('This import is not writable.');
|
||||
await receiveGameFile(stagingRoot(id), file, body);
|
||||
}
|
||||
|
||||
const publishing = new Map<string, Promise<GameLibraryVersion>>();
|
||||
export async function publishImport(hostId: string, id: string): Promise<GameLibraryVersion> {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
if (version.state === 'ready') return version;
|
||||
if (version.state !== 'importing' || !version.files.length || !version.manifestSha256) throw new Error('Import needs a complete manifest before publication.');
|
||||
const existing = publishing.get(id); if (existing) return existing;
|
||||
const task = (async () => {
|
||||
let root = stagingRoot(id);
|
||||
// A restart between the atomic directory rename and metadata commit is recoverable.
|
||||
try { if ((await stat(version.storageLocation)).isDirectory()) root = version.storageLocation; }
|
||||
catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; }
|
||||
for (const file of version.files) await verifyGameFile(root, file);
|
||||
if (root !== version.storageLocation) {
|
||||
await mkdir(dirname(version.storageLocation), { recursive: true });
|
||||
await rename(root, version.storageLocation);
|
||||
}
|
||||
database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id);
|
||||
return getLibraryVersion(id)!;
|
||||
})();
|
||||
publishing.set(id, task);
|
||||
try { return await task; } finally { publishing.delete(id); }
|
||||
}
|
||||
|
||||
/** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */
|
||||
export function importPublicationStatus(hostId: string, id: string) {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
return {
|
||||
status: version.state === 'ready' ? 'published' as const : version.state === 'failed' ? 'failed' as const : publishing.has(id) ? 'verifying' as const : 'pending' as const,
|
||||
version: { id: version.id, game: version.game, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 },
|
||||
error: version.error
|
||||
};
|
||||
}
|
||||
export function requestImportPublication(hostId: string, id: string) {
|
||||
const version = requireImportOwner(hostId, id);
|
||||
if (version.state === 'importing' && !publishing.has(id)) {
|
||||
if (!version.files.length || !version.manifestSha256) throw new Error('Import requires a manifest before publication.');
|
||||
void publishImport(hostId, id).catch(cause => {
|
||||
database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'")
|
||||
.run((cause instanceof Error ? cause.message : 'Library publication failed.').slice(0, 2000), id);
|
||||
});
|
||||
}
|
||||
return importPublicationStatus(hostId, id);
|
||||
}
|
||||
|
||||
export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void {
|
||||
if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.');
|
||||
const handle = database();
|
||||
const now = new Date().toISOString();
|
||||
handle.transaction(() => {
|
||||
const reported = new Set<string>();
|
||||
for (const raw of input) {
|
||||
if (!raw || typeof raw !== 'object') throw new Error('Invalid library cache report.');
|
||||
const id = raw.libraryVersionId ?? raw.id;
|
||||
if (typeof id !== 'string') throw new Error('Invalid library version ID.');
|
||||
if (reported.has(id)) throw new Error('Duplicate version in host cache report.');
|
||||
reported.add(id);
|
||||
const version = getLibraryVersion(id); if (!version || version.state !== 'ready') continue;
|
||||
let status = raw.status ?? raw.state;
|
||||
if (!['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes(status)) throw new Error('Invalid library cache state.');
|
||||
const checksum = typeof raw.manifestSha256 === 'string' ? raw.manifestSha256 : null;
|
||||
let error = typeof raw.error === 'string' ? raw.error.slice(0, 2000) : null;
|
||||
if (['ready', 'outdated'].includes(status) && checksum !== version.manifestSha256) { status = 'failed'; error = 'Host manifest checksum does not match the published version.'; }
|
||||
const bytes = raw.bytesDownloaded ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0);
|
||||
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) throw new Error('Invalid library download progress.');
|
||||
if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { status = 'failed'; error = 'Host reported an incomplete library as verified.'; }
|
||||
handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status,
|
||||
bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at`)
|
||||
.run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now);
|
||||
}
|
||||
if (options.snapshot) {
|
||||
const cached = handle.prepare('SELECT version_id FROM host_library_caches WHERE host_id=?').all(hostId) as { version_id: string }[];
|
||||
for (const cache of cached) if (!reported.has(cache.version_id)) {
|
||||
handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=? AND version_id=?")
|
||||
.run(now, hostId, cache.version_id);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
export function queueLibraryDownload(hostId: string, versionId: string) {
|
||||
const version = getLibraryVersion(versionId);
|
||||
if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
|
||||
const host = getHost(hostId);
|
||||
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
|
||||
if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.');
|
||||
const prior = getHostLibraryCache(hostId, versionId);
|
||||
if (prior?.operationId) {
|
||||
const operation = getHostOperation(prior.operationId);
|
||||
if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) return operation;
|
||||
}
|
||||
const operation = queueHostOperation(hostId, 'library.download', {
|
||||
libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`,
|
||||
filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`,
|
||||
manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes
|
||||
});
|
||||
updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: 0 }]);
|
||||
database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId);
|
||||
return operation;
|
||||
}
|
||||
|
||||
export function reconcileLibraryOperations(): void {
|
||||
const handle = database();
|
||||
for (const row of handle.prepare("SELECT id,operation_id FROM game_file_versions WHERE state='importing' AND operation_id IS NOT NULL").all() as { id: string; operation_id: string }[]) {
|
||||
const operation = getHostOperation(row.operation_id);
|
||||
if (operation?.state === 'failed') handle.prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'").run(operation.error ?? 'Host import failed.', row.id);
|
||||
}
|
||||
for (const cache of handle.prepare("SELECT * FROM host_library_caches WHERE status IN ('downloading','verifying') AND operation_id IS NOT NULL").all() as CacheRow[]) {
|
||||
const operation = getHostOperation(cache.operation_id!);
|
||||
if (operation?.state === 'failed') handle.prepare("UPDATE host_library_caches SET status='failed',error=?,updated_at=? WHERE host_id=? AND version_id=?").run(operation.error ?? 'Host download failed.', new Date().toISOString(), cache.host_id, cache.version_id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Lightweight list used in launch forms; large manifests stay on the detail endpoint. */
|
||||
export function libraryOverview() {
|
||||
const hosts = listHosts();
|
||||
return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
|
||||
...version, caches: hosts.map(host => {
|
||||
const cached = getHostLibraryCache(host.id, version.id);
|
||||
const latest = library.versions.find(item => item.state === 'ready');
|
||||
if (cached) return { ...cached, status: cached.status === 'ready' && latest?.id !== version.id ? 'outdated' as const : cached.status };
|
||||
return {
|
||||
hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,
|
||||
bytesTotal: version.sizeBytes, manifestSha256: null, error: null, updatedAt: version.createdAt, operationId: null
|
||||
};
|
||||
})
|
||||
})) })), hosts };
|
||||
}
|
||||
Reference in New Issue
Block a user