josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify game files to automatic T5 and T6 uploads

Browse Source
This commit is contained in:
josh
2026-10-04 12:23:36 -04:00
parent 482e27e735
commit 68323c0baa
19 changed files with 588 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/game-files/index.ts
+36 -36
View File
@@ -7,6 +7,7 @@ import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hos
import { ensureGameFileSchema } from './schema.js';
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
@@ -35,23 +36,29 @@ export function getLibraryFile(versionId: string, path: string) {
.get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null;
}
export function newestReadyLibrary(game: string): GameLibraryVersion | null {
const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
return row ? versionFromRow(row) : null;
}
/** Preparation needs only the latest published version of each library, never its full manifest. */
/** Hosts follow the latest successful publication for each game, including legacy libraries. */
export function latestPublishedLibraries() {
return database().prepare(`SELECT v.id, l.name, l.game, v.version, v.size_bytes AS sizeBytes, v.manifest_sha256 AS manifestSha256
const latest = database().prepare(`SELECT v.id, l.game, v.version, v.size_bytes AS sizeBytes, v.manifest_sha256 AS manifestSha256, v.published_at AS publishedAt
FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE v.state='ready' AND v.id=(SELECT newest.id FROM game_file_versions newest
WHERE newest.library_id=l.id AND newest.state='ready' ORDER BY newest.created_at DESC,newest.rowid DESC LIMIT 1)
ORDER BY l.name`).all() as { id: string; name: string; game: GameLibraryGame; version: string; sizeBytes: number; manifestSha256: string }[];
WHERE v.state='ready' AND l.game=?
ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC LIMIT 1`);
return GAME_FILE_GAMES.flatMap(game => {
const version = latest.get(game.id) as { id: string; game: GameLibraryGame; version: string; sizeBytes: number; manifestSha256: string; publishedAt: string | null } | undefined;
return version ? [{ ...version, name: game.name }] : [];
});
}
export function listLibraries(): GameLibrary[] {
reconcileLibraryOperations();
return (database().prepare('SELECT * FROM game_file_libraries ORDER BY name').all() as Array<{ id: string; name: string; game: GameLibraryGame; description: string; created_at: string }>).map(row => ({
id: row.id, name: row.name, game: row.game, description: row.description, createdAt: row.created_at,
versions: (database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.id=? ORDER BY v.created_at DESC, v.rowid DESC').all(row.id) as VersionRow[]).map(versionFromRow)
}));
const latest = new Map(latestPublishedLibraries().map(version => [version.game, version.id]));
return GAME_FILE_GAMES.map(game => {
const versions = (database().prepare(`SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
WHERE l.game=? ORDER BY v.created_at DESC,v.rowid DESC`).all(game.id) as VersionRow[]).map(versionFromRow);
const firstLibrary = database().prepare('SELECT created_at FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game.id) as { created_at: string } | undefined;
return { id: game.id, name: game.name, game: game.id, description: '', createdAt: firstLibrary?.created_at ?? '', latestVersionId: latest.get(game.id) ?? null, versions };
});
}
export function getHostLibraryCache(hostId: string, versionId: string): HostLibraryCache | null {
const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined;
@@ -70,27 +77,29 @@ function textField(input: unknown, label: string, max: number, required = true):
return input.trim();
}
/** Reuse a game's original storage parent without rewriting immutable historical versions. */
function libraryForGame(game: GameLibraryGame, now: string): string {
const handle = database();
const existing = handle.prepare('SELECT id FROM game_file_libraries WHERE game=? ORDER BY created_at,rowid LIMIT 1').get(game) as { id: string } | undefined;
if (existing) return existing.id;
const id = randomUUID();
// The legacy name column is globally unique, so its private value must not collide with an old user label.
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(id, `game-files:${game}:${id}`, game, '', now);
return id;
}
/** Register a folder on the controller; no host or Docker volume is needed. */
export function createLibraryUpload(input: Record<string, unknown>): GameLibraryVersion {
const game = input.game;
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
const name = textField(input.name, 'library name', 160);
const description = textField(input.description ?? '', 'description', 4000, false);
const version = textField(input.version, 'version label', 120);
const files = validateUploadPlan(input.files);
const handle = database();
const id = randomUUID(); const now = new Date().toISOString();
handle.transaction(() => {
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
if (!library) {
library = { id: randomUUID(), game };
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
}
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
const libraryId = libraryForGame(game, now);
// Keep legacy non-null columns for existing databases and published versions.
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume)
VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, library.id, version, description, files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id));
VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, libraryId, id, '', files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id));
const insert = handle.prepare('INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)');
for (const file of files) insert.run(id, file.path, file.size, file.lastModified);
})();
@@ -181,22 +190,13 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
const hostId = textField(input.hostId, 'source host', 128);
const host = getHost(hostId);
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled source host.');
const name = textField(input.name, 'library name', 160);
const description = textField(input.description ?? '', 'description', 4000, false);
const version = textField(input.version, 'version label', 120);
const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
const handle = database();
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
const id = randomUUID(); const now = new Date().toISOString();
handle.transaction(() => {
if (!library) {
library = { id: randomUUID(), game };
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
}
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
const libraryId = libraryForGame(game, now);
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, libraryId, id, '', '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
})();
try {
const operation = queueHostOperation(hostId, 'library.import', {
@@ -318,7 +318,7 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options:
})();
}
export function queueLibraryDownload(hostId: string, versionId: string) {
export function queueLibraryDownload(hostId: string, versionId: string, options: { automatic?: boolean } = {}) {
const version = getLibraryVersion(versionId);
if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
const host = getHost(hostId);
@@ -333,7 +333,8 @@ export function queueLibraryDownload(hostId: string, versionId: string) {
const operation = queueHostOperation(hostId, 'library.download', {
libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`,
filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`,
manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes
manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes,
...(options.automatic ? { automatic: true } : {})
});
updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: retainedBytes, manifestSha256: prior?.manifestSha256 }]);
database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId);
@@ -358,8 +359,7 @@ export function libraryOverview() {
return { libraries: listLibraries().map(library => ({ ...library, versions: library.versions.map(({ files: _files, ...version }) => ({
...version, caches: hosts.map(host => {
const cached = getHostLibraryCache(host.id, version.id);
const latest = library.versions.find(item => item.state === 'ready');
if (cached) return { ...cached, status: cached.status === 'ready' && latest?.id !== version.id ? 'outdated' as const : cached.status };
if (cached) return { ...cached, status: cached.status === 'ready' && library.latestVersionId !== version.id ? 'outdated' as const : cached.status };
return {
hostId: host.id, libraryVersionId: version.id, status: 'missing' as const, bytesDownloaded: 0,
bytesTotal: version.sizeBytes, manifestSha256: null, error: null, updatedAt: version.createdAt, operationId: null
src/lib/server/hosts/preparation.ts
+25 -5
View File
@@ -5,6 +5,7 @@ import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } f
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
@@ -53,8 +54,8 @@ function targets(host: Host): Target[] {
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
item: { kind: 'library', label: `${version.name} · ${version.version}`, state: ready ? 'ready' : 'waiting',
detail: ready ? 'Game files are downloaded and verified.' : 'Game files will be downloaded and verified automatically.',
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
detail: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes } });
}
const blocker = hostBlocker(host);
@@ -82,7 +83,7 @@ function targets(host: Host): Target[] {
export function hostPreparationStatus(host: Host): HostPreparation {
const items = targets(host).map(target => target.item);
if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
detail: 'Upload and publish a game-file library once; hosts download it automatically.' });
detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
const state = items.some(item => item.state === 'blocked') ? 'blocked'
: items.some(item => item.state === 'preparing') ? 'preparing'
: items.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
@@ -97,25 +98,44 @@ export function presentHostPreparation(host: Host): Host {
return { ...host, preparation };
}
function supersedeUndeliveredDownloads(hostId: string): void {
const latest = new Set(latestPublishedLibraries().map(version => version.id));
const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending'
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[];
const now = new Date().toISOString();
for (const operation of pending) {
if (latest.has(operation.versionId)) continue;
// Delivered operations may already be writing files. Only unsubmitted
// automatic work can be replaced without consulting the agent journal.
db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload);
db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')")
.run(now, hostId, operation.versionId, operation.id);
}
}
/** Agent heartbeats and polls drive durable preparation; no open UI or in-memory worker is needed. */
export function reconcileHostPreparation(hostId: string): HostPreparation {
return db().transaction(() => {
const host = getHost(hostId);
if (!host) throw new Error('Host was not found.');
if (!hostBlocker(host)) {
supersedeUndeliveredDownloads(hostId);
// Queue only one setup mutation at a time so each capacity check uses
// fresh disk inventory. This also respects manually queued downloads.
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed')
ORDER BY updated_at DESC LIMIT 1`).get(hostId) as { updated_at: string } | undefined;
AND (error IS NULL OR error != ?)
ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined;
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
// A failed or oversized library must not prevent other libraries
// from preparing. The runtime still comes first when eligible.
const target = targets(host).find(target => target.item.state === 'waiting');
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId);
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
}
}
return hostPreparationStatus(host);
src/lib/server/provisioning/native.ts
+7 -5
View File
@@ -4,7 +4,7 @@ import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
import { getLibraryVersion, requireReadyLibrary } from '../game-files/index.js';
import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -29,9 +29,11 @@ interface FilePlan { id: number; path: string; contentBase64: string | null; sha
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
if (typeof input.libraryId !== 'string' || !input.libraryId) throw new Error('Select a published game-file version.');
const version = getLibraryVersion(input.libraryId);
if (!version || version.game !== game || version.state !== 'ready') throw new Error('Choose a published game-file version for the selected game.');
const version = newestReadyLibrary(game);
if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`);
}
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
@@ -68,7 +70,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
const checks: LaunchPreview['checks'] = [
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
{ label: 'Game files', state: 'pass', detail: `Immutable library version ${selection.libraryId} is ready on the selected host.` },
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` },
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }