From 596b160be36b36a64d9148255dc2cd15736a9b0b Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Fri, 9 Oct 2026 17:15:15 -0400 Subject: [PATCH] Restrict launches to prepared runtimes (Refs #93) --- agent/ec_agent.py | 14 +- agent/tests/test_agent.py | 54 +++++++- docs/guides/native-hosts.md | 13 +- scripts/fixture-data.ts | 2 +- src/lib/hosts.ts | 3 + src/lib/server/hosts/availability.ts | 2 +- src/lib/server/hosts/capacity-overview.ts | 2 +- src/lib/server/hosts/index.ts | 20 +-- src/lib/server/hosts/preparation.ts | 2 +- src/lib/server/hosts/request-context.ts | 16 ++- src/lib/server/provisioning/native.ts | 4 +- .../ui/components/NativeHostSelection.svelte | 2 +- src/routes/api/system-status/+server.ts | 4 +- src/routes/hosts/[id]/+page.svelte | 2 +- static/agent/ec-agent.py | 14 +- static/agent/manifest.json | 4 +- tests/helpers/native-host.ts | 2 +- tests/unit/agent-operation-delivery.test.ts | 2 +- tests/unit/capacity-overview.test.ts | 2 +- .../collection-launch-performance.test.ts | 2 +- .../unit/collection-match-performance.test.ts | 2 +- .../unit/game-file-transfer-progress.test.ts | 2 +- tests/unit/host-agent-lanes.test.ts | 2 +- tests/unit/host-availability.test.ts | 28 +++- tests/unit/host-capacity-calibration.test.ts | 2 +- tests/unit/host-observation-retention.test.ts | 2 +- tests/unit/host-preparation.test.ts | 97 ++++++++------ tests/unit/host-request-cost.test.ts | 2 +- tests/unit/native-creation-intents.test.ts | 2 +- tests/unit/native-hosts.test.ts | 23 +++- tests/unit/native-launch-preparation.test.ts | 2 +- tests/unit/native-provisioning.test.ts | 2 +- tests/unit/native-safety.test.ts | 2 +- tests/unit/rebalancing-runs.test.ts | 4 +- tests/unit/retention.test.ts | 2 +- tests/unit/runtime-image-policy.test.ts | 123 +++++++++++++++--- 36 files changed, 347 insertions(+), 116 deletions(-) diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 435dde8..e7969c5 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.8.5" +VERSION = "1.8.6" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1462,7 +1462,7 @@ class Agent: progress = list(progress.values()) libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress) caches.extend(progress) - images = [] + images, prepared_images = [], [] if docker_running: with contextlib.suppress(AgentError): lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines() @@ -1480,7 +1480,13 @@ class Agent: aliases = self.root / "agent" / "runtime-images.json" if aliases.exists(): with contextlib.suppress(OSError, ValueError): - images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) + saved = json.loads(aliases.read_text()) + if isinstance(saved, dict): + # Only runtime.pull records administrator-prepared + # references. Other Docker cache entries are useful + # diagnostics, but do not grant launch permission. + prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids] + images.extend(prepared_images) resources = {} if docker_running: observations = [] @@ -1499,7 +1505,7 @@ class Agent: update = self.update_inventory() capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) plutonium_cache = self.plutonium_cache_inventory() - return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False): health = self.inspect(instance_id, generation) diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 01a68d9..1a7e591 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -2502,12 +2502,62 @@ class AgentTests(unittest.TestCase): return f"ghcr.io/ayymoss/plutainer|latest|{image_digest}|{image_id}\n" return original(*args, **kwargs) self.docker.call = call - module.atomic_json(self.agent.root / "agent/runtime-images.json", {"ghcr.io/ayymoss/plutainer": image_id, "removed-image": "missing-id"}) - images = self.agent.inventory()["runtimeImages"] + prepared = ["ghcr.io/ayymoss/plutainer", "ghcr.io/ayymoss/plutainer@" + image_digest] + module.atomic_json(self.agent.root / "agent/runtime-images.json", {**dict.fromkeys(prepared, image_id), "removed-image": "missing-id"}) + inventory = self.agent.inventory() + images = inventory["runtimeImages"] self.assertIn("ghcr.io/ayymoss/plutainer@" + image_digest, images) self.assertIn("ghcr.io/ayymoss/plutainer:latest", images) self.assertIn("ghcr.io/ayymoss/plutainer", images) self.assertNotIn("removed-image", images) + self.assertEqual(inventory["preparedRuntimeImages"], prepared) + + def test_runtime_pull_prepares_only_the_requested_reference_from_the_docker_cache(self): + image = "registry.example/custom-runtime:stable" + image_id = "sha256:" + "1" * 64 + original = self.docker.call + + def call(*args, **kwargs): + if args[:2] == ("image", "ls"): + return (f"registry.example/custom-runtime|stable||{image_id}\n" + f"registry.example/custom-runtime|unprepared-alias||{image_id}\n" + "unrelated-service|latest||sha256:unrelated\n") + if args[:2] == ("image", "inspect"): + self.assertEqual(args[-1], image) + return image_id + return original(*args, **kwargs) + + self.docker.call = call + before = self.agent.inventory() + self.assertIn(image, before["runtimeImages"]) + self.assertIn("unrelated-service:latest", before["runtimeImages"]) + self.assertEqual(before["preparedRuntimeImages"], []) + + result = self.agent.execute(self.operation("runtime.pull", {"image": image})) + self.assertEqual(result["state"], "completed") + inventory = self.agent.inventory() + self.assertEqual(inventory["runtimeImages"], before["runtimeImages"]) + self.assertEqual(inventory["preparedRuntimeImages"], [image]) + + def test_runtime_preparation_inventory_fails_closed_when_records_or_docker_are_unavailable(self): + aliases = self.agent.root / "agent/runtime-images.json" + original = self.docker.call + + def call(*args, **kwargs): + if args[:2] == ("image", "ls"): + return "custom-runtime|latest||sha256:prepared\n" + return original(*args, **kwargs) + + self.docker.call = call + for content in ("invalid json", "[]", '{"custom-runtime:latest": []}', '{"custom-runtime:latest": "sha256:removed"}'): + with self.subTest(content=content): + aliases.write_text(content) + inventory = self.agent.inventory() + self.assertEqual(inventory["preparedRuntimeImages"], []) + self.assertEqual(inventory["runtimeImages"], ["custom-runtime:latest"]) + module.atomic_json(aliases, {"custom-runtime:latest": "sha256:prepared"}) + with patch.object(self.docker, "call", side_effect=AgentError("Docker unavailable")): + self.assertEqual(self.agent.inventory()["preparedRuntimeImages"], []) def test_heartbeat_applies_configured_host_port_range_before_deployment(self): self.agent.request = lambda *_: {"host": {"portMin": 30000, "portMax": 30100}} diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index ee55bc0..893fb43 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -243,10 +243,15 @@ servers. Existing API clients may still explicitly constrain a launch to a host. Queued jobs retain their accepted game-file version. Automatic host preparation also downloads the game files needed by unplaced jobs; a newer upload does not remove files still needed by the queue. Only the configured default runtime is -pulled automatically. Custom images must already be cached on the selected host, -or on an active host for automatic placement, before a launch is accepted. Admins -can prepare them from the host page. Queued launches and migrations never trigger -custom image pulls, and container creation cannot implicitly pull a missing image. +pulled automatically. Custom images must have been prepared by an admin from the +selected host's page, or on an active host for automatic placement, before a launch +is accepted. An image cached by another Docker workload does not qualify. Agent +1.8.6 reports these exact prepared references from `runtime-images.json` while +their image IDs remain cached; the controller also trusts its configured default +when cached. Host pages and APIs expose only prepared runtimes. Older agents can +still run the configured default, but must update before accepting custom runtimes. +Queued launches and migrations never trigger custom image pulls, and container +creation cannot implicitly pull a missing image. ### Resource samples and learned allowances diff --git a/scripts/fixture-data.ts b/scripts/fixture-data.ts index e54d559..24c785d 100644 --- a/scripts/fixture-data.ts +++ b/scripts/fixture-data.ts @@ -163,7 +163,7 @@ export function seedFixtureData(handle: Database.Database, now: number): void { storageAvailable: 240 * GIB, dockerRunning: true, directoriesWritable: true, - runtimeImages: [image], + preparedRuntimeImages: [image], libraries: [ { id: 'fixture-t5', state: 'ready' }, { id: 'fixture-t6', state: 'ready' } diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index b47e158..65a3d05 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -156,7 +156,10 @@ export interface HostInventory { dockerVersion?: string; dockerRunning?: boolean; directoriesWritable?: boolean; + /** Agent-reported Docker cache; removed from presented host inventory. */ runtimeImages?: string[]; + /** Prepared, locally available image references eligible for launches. */ + preparedRuntimeImages?: string[]; libraries?: HostLibrary[]; currentOperation?: string | null; [key: string]: unknown; diff --git a/src/lib/server/hosts/availability.ts b/src/lib/server/hosts/availability.ts index dd4dbb8..f9da4b2 100644 --- a/src/lib/server/hosts/availability.ts +++ b/src/lib/server/hosts/availability.ts @@ -198,7 +198,7 @@ export function launchAvailability(selections: NativePlacement[]): { ); const eligible = (candidate: HostAvailability, selection: NativePlacement) => (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) && - !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') && + !!candidate.host.inventory.preparedRuntimeImages?.includes(selection.image ?? '') && !!candidate.host.inventory.libraries?.some( (library) => library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state) diff --git a/src/lib/server/hosts/capacity-overview.ts b/src/lib/server/hosts/capacity-overview.ts index df6b5a1..a62c655 100644 --- a/src/lib/server/hosts/capacity-overview.ts +++ b/src/lib/server/hosts/capacity-overview.ts @@ -66,7 +66,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise games.includes(game))) reasons.push('Latest game files for this selection are not ready.'); diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index ee9f7d7..46317f0 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -140,7 +140,7 @@ function hostFromRow(row: HostRow): Host { ) reasons.push('Insufficient available game storage.'); const infrastructureReady = reasons.length === 0; - const runtimeReady = !!inventory.runtimeImages?.length; + const runtimeReady = !!inventory.preparedRuntimeImages?.length; const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready'); if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.'); if (!libraryReady) reasons.push('No verified game-file library is available yet.'); @@ -676,12 +676,14 @@ export function validateInventory(value: unknown): HostInventory { for (const key of ['dockerRunning', 'directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); - if ( - inventory.runtimeImages !== undefined && - (!Array.isArray(inventory.runtimeImages) || - inventory.runtimeImages.some((image) => typeof image !== 'string')) - ) - throw new Error('Invalid runtime image inventory.'); + for (const key of ['runtimeImages', 'preparedRuntimeImages'] as const) { + const images = inventory[key]; + if ( + images !== undefined && + (!Array.isArray(images) || images.some((image) => typeof image !== 'string')) + ) + throw new Error('Invalid runtime image inventory.'); + } if ( inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || @@ -952,8 +954,8 @@ export function requireReadyHost( ) ) reasons.push('The selected game library is not ready on this host.'); - if (options.image && !host.inventory.runtimeImages?.includes(options.image)) - reasons.push('The selected runtime image is not cached on this host.'); + if (options.image && !host.inventory.preparedRuntimeImages?.includes(options.image)) + reasons.push('The selected runtime image is not prepared on this host.'); if ( options.port !== undefined && !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol) diff --git a/src/lib/server/hosts/preparation.ts b/src/lib/server/hosts/preparation.ts index a85a299..607bc97 100644 --- a/src/lib/server/hosts/preparation.ts +++ b/src/lib/server/hosts/preparation.ts @@ -190,7 +190,7 @@ function targets(host: Host): Target[] { const result: Target[] = [config.defaultRuntimeImage].map((image) => { const operation = latestOperation(host.id, 'runtime.pull', 'image', image); const runtimeReady = - host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); + host.inventory.preparedRuntimeImages?.includes(image) && verifiedThisEnable(operation); return { image, requiredBytes: host.minimumFreeBytes, diff --git a/src/lib/server/hosts/request-context.ts b/src/lib/server/hosts/request-context.ts index 23087d2..fe9cbc1 100644 --- a/src/lib/server/hosts/request-context.ts +++ b/src/lib/server/hosts/request-context.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from 'node:async_hooks'; +import { config } from '../config.js'; import type { HostInventory } from '../../hosts.js'; type InventoryCache = Map; @@ -21,7 +22,20 @@ export function parseHostInventory(hostId: string, json: string): HostInventory const cache = requests.getStore()?.inventories; const saved = cache?.get(hostId); if (saved?.json === json) return saved.inventory; - const inventory: HostInventory = JSON.parse(json); + // Full Docker inventory can include unrelated workloads. Keep it out of + // every presented host, including inventory persisted by older agents. + const { runtimeImages, ...inventory }: HostInventory = JSON.parse(json); + // The configured default is explicitly trusted by the controller. Older + // agents may prove it is cached, but cannot authorize any custom runtime. + const reported = inventory.preparedRuntimeImages; + const prepared = new Set( + Array.isArray(reported) && reported.every((image) => typeof image === 'string') + ? reported + : [] + ); + if (runtimeImages?.includes(config.defaultRuntimeImage)) + prepared.add(config.defaultRuntimeImage); + inventory.preparedRuntimeImages = [...prepared]; // Compare the stored JSON so heartbeats and transaction rollbacks within // a request remain visible. Other host state is always read afresh. cache?.set(hostId, { json, inventory }); diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 63e7e13..988e19d 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -136,7 +136,7 @@ export function nativeLaunchSelection( throw new Error('The selected host is being removed. Choose another host.'); } // Only the controller-configured default may be prepared automatically. - // Custom runtimes must already be present in trusted host inventory; a + // Custom runtimes must be reported as admin-prepared by the host agent; a // launch request must never authorize a registry download. if ( image !== config.defaultRuntimeImage && @@ -144,7 +144,7 @@ export function nativeLaunchSelection( (host) => !host.decommission && (automatic ? host.mode === 'active' : host.id === input.hostId) && - host.inventory.runtimeImages?.includes(image) + host.inventory.preparedRuntimeImages?.includes(image) ) ) { throw new Error( diff --git a/src/lib/ui/components/NativeHostSelection.svelte b/src/lib/ui/components/NativeHostSelection.svelte index 1a9e7c7..3adc94b 100644 --- a/src/lib/ui/components/NativeHostSelection.svelte +++ b/src/lib/ui/components/NativeHostSelection.svelte @@ -45,7 +45,7 @@ hosts.filter( (host) => host.readiness === 'ready' && - host.inventory.runtimeImages?.includes(selection.image) + host.inventory.preparedRuntimeImages?.includes(selection.image) ) ); const latest = $derived(latestGameFiles(libraries)); diff --git a/src/routes/api/system-status/+server.ts b/src/routes/api/system-status/+server.ts index 710a753..9114182 100644 --- a/src/routes/api/system-status/+server.ts +++ b/src/routes/api/system-status/+server.ts @@ -31,7 +31,7 @@ export async function GET() { cache.status === 'ready' && readyHosts.some((host) => host.id === cache.hostId) ) ); - const runtimes = connected.filter((host) => host.inventory.runtimeImages?.length); + const runtimes = connected.filter((host) => host.inventory.preparedRuntimeImages?.length); const checks: SetupCheck[] = [ { id: 'hosts', @@ -46,7 +46,7 @@ export async function GET() { title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length - ? `${runtimes.length} connected hosts report cached runtime images.` + ? `${runtimes.length} connected hosts report prepared runtime images.` : 'Hosts prepare the configured Plutainer runtime automatically.', href: '/hosts', action: 'View host setup' diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index 01ade5c..24190f9 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -447,7 +447,7 @@ {/each} {:else}

No cached game libraries reported yet.

{/if}

- Cached images: {host.inventory.runtimeImages?.join(', ') || + Prepared runtimes: {host.inventory.preparedRuntimeImages?.join(', ') || 'None reported yet'}

{#if canManage}
diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 435dde8..e7969c5 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.8.5" +VERSION = "1.8.6" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1462,7 +1462,7 @@ class Agent: progress = list(progress.values()) libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress) caches.extend(progress) - images = [] + images, prepared_images = [], [] if docker_running: with contextlib.suppress(AgentError): lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines() @@ -1480,7 +1480,13 @@ class Agent: aliases = self.root / "agent" / "runtime-images.json" if aliases.exists(): with contextlib.suppress(OSError, ValueError): - images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) + saved = json.loads(aliases.read_text()) + if isinstance(saved, dict): + # Only runtime.pull records administrator-prepared + # references. Other Docker cache entries are useful + # diagnostics, but do not grant launch permission. + prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids] + images.extend(prepared_images) resources = {} if docker_running: observations = [] @@ -1499,7 +1505,7 @@ class Agent: update = self.update_inventory() capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) plutonium_cache = self.plutonium_cache_inventory() - return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False): health = self.inspect(instance_id, generation) diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 166f11d..3cdb700 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.8.5", + "version": "1.8.6", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "56187a244b551af884db6bd6419daf7dd7d931fa1e5d5c5414559bb6e25f1094", + "sha256": "79702dc8cf80d2521153bfda9241f7240285014fe746fc921f0a7793dd81212b", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/helpers/native-host.ts b/tests/helpers/native-host.ts index b3f298f..6a7f2eb 100644 --- a/tests/helpers/native-host.ts +++ b/tests/helpers/native-host.ts @@ -16,7 +16,7 @@ export async function prepareNativeHost(servers: string[] = []) { dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, - runtimeImages: [config.defaultRuntimeImage] + preparedRuntimeImages: [config.defaultRuntimeImage] }; hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); const versions = new Map(); diff --git a/tests/unit/agent-operation-delivery.test.ts b/tests/unit/agent-operation-delivery.test.ts index 27b3029..f67a354 100644 --- a/tests/unit/agent-operation-delivery.test.ts +++ b/tests/unit/agent-operation-delivery.test.ts @@ -10,7 +10,7 @@ const inventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['plutainer:test'], + preparedRuntimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }], capabilities: ['instance.cancel-create', 'operations.health-lane'] }; diff --git a/tests/unit/capacity-overview.test.ts b/tests/unit/capacity-overview.test.ts index b9343c7..f0f6e2d 100644 --- a/tests/unit/capacity-overview.test.ts +++ b/tests/unit/capacity-overview.test.ts @@ -18,7 +18,7 @@ const inventory = { cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, - runtimeImages: [image], + preparedRuntimeImages: [image], libraries: [ { id: 't5-version', state: 'ready' }, { id: 't6-version', state: 'ready' } diff --git a/tests/unit/collection-launch-performance.test.ts b/tests/unit/collection-launch-performance.test.ts index 3144e85..935ee36 100644 --- a/tests/unit/collection-launch-performance.test.ts +++ b/tests/unit/collection-launch-performance.test.ts @@ -35,7 +35,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, - runtimeImages: [image] + preparedRuntimeImages: [image] }; enrollHost(createEnrollment(hostId).token, inventory); const { createLibraryImport, updateHostLibraryCaches } = diff --git a/tests/unit/collection-match-performance.test.ts b/tests/unit/collection-match-performance.test.ts index b733237..32507bb 100644 --- a/tests/unit/collection-match-performance.test.ts +++ b/tests/unit/collection-match-performance.test.ts @@ -47,7 +47,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, - runtimeImages: [image] + preparedRuntimeImages: [image] }; agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential; const { createLibraryImport, updateHostLibraryCaches } = diff --git a/tests/unit/game-file-transfer-progress.test.ts b/tests/unit/game-file-transfer-progress.test.ts index 960fe1e..fc044d1 100644 --- a/tests/unit/game-file-transfer-progress.test.ts +++ b/tests/unit/game-file-transfer-progress.test.ts @@ -33,7 +33,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: [runtime] + preparedRuntimeImages: [runtime] }); version = await publish(); }); diff --git a/tests/unit/host-agent-lanes.test.ts b/tests/unit/host-agent-lanes.test.ts index 3ef0863..7c412d2 100644 --- a/tests/unit/host-agent-lanes.test.ts +++ b/tests/unit/host-agent-lanes.test.ts @@ -8,7 +8,7 @@ const inventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['image'], + preparedRuntimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] }; const lane: HostAgentLane = { diff --git a/tests/unit/host-availability.test.ts b/tests/unit/host-availability.test.ts index e5c0314..8273390 100644 --- a/tests/unit/host-availability.test.ts +++ b/tests/unit/host-availability.test.ts @@ -15,7 +15,7 @@ const inventory = { cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, - runtimeImages: ['image'], + preparedRuntimeImages: ['image'], libraries: [ { id: 't5-library', state: 'ready' }, { id: 't6-library', state: 'ready' } @@ -46,6 +46,32 @@ afterEach(async () => { }); describe('cached launch availability', () => { + it.each([{ preparedRuntimeImages: [] }, { preparedRuntimeImages: ['another:image'] }])( + 'excludes a cached image until it is prepared ($preparedRuntimeImages)', + async ({ preparedRuntimeImages }) => { + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { launchAvailability } = + await import('../../src/lib/server/hosts/availability.js'); + heartbeat(hostId, { + ...inventory, + runtimeImages: ['image'], + preparedRuntimeImages + }); + const unavailable = launchAvailability([t5]); + expect(unavailable.availability).toMatchObject({ + estimatedAvailable: 0, + portsAvailable: 0 + }); + expect(unavailable.warnings.join(' ')).toContain('Waiting for an eligible host'); + + heartbeat(hostId, { ...inventory, runtimeImages: ['image'] }); + expect(launchAvailability([t5]).availability).toMatchObject({ + estimatedAvailable: 1, + portsAvailable: 3 + }); + } + ); + it('reads cached estimates without placement, historical scans, or repeated database reads', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const capacity = await import('../../src/lib/server/hosts/capacity.js'); diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index 095f9f8..ce756a2 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -14,7 +14,7 @@ const inventory = { protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB, - runtimeImages: ['image'], + preparedRuntimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] }; const sample = { diff --git a/tests/unit/host-observation-retention.test.ts b/tests/unit/host-observation-retention.test.ts index 00fdcfe..0140076 100644 --- a/tests/unit/host-observation-retention.test.ts +++ b/tests/unit/host-observation-retention.test.ts @@ -9,7 +9,7 @@ const inventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['plutainer:test'], + preparedRuntimeImages: ['plutainer:test'], libraries: [{ id: 'library', state: 'ready' }] }; let fixture: Fixture; diff --git a/tests/unit/host-preparation.test.ts b/tests/unit/host-preparation.test.ts index 88d8c39..c29d46d 100644 --- a/tests/unit/host-preparation.test.ts +++ b/tests/unit/host-preparation.test.ts @@ -18,7 +18,7 @@ const inventory: HostInventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: [], + preparedRuntimeImages: [], libraries: [], capabilities: ['host.decommission'] }; @@ -123,7 +123,7 @@ describe('automatic host preparation', () => { const t5 = await publish('t5'); const t6 = await publish('t6'); const readyInventory = { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })), capabilities: [ ...inventory.capabilities!, @@ -195,7 +195,7 @@ describe('automatic host preparation', () => { it('requires both games when enabling even if one library is already cached', async () => { const version = await publish('t5'); const id = enroll({ - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); verified(id, version); @@ -212,7 +212,7 @@ describe('automatic host preparation', () => { const t5 = await publish('t5'); const t6 = await publish('t6'); const readyInventory = { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })) }; const id = enroll(readyInventory); @@ -310,7 +310,11 @@ describe('automatic host preparation', () => { expect(operations(host.id)).toHaveLength(1); vi.setSystemTime(Date.now() + 1); - const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] }; + const preparedInventory = { + ...inventory, + preparedRuntimeImages: [runtime], + libraryCaches: [] + }; const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) }); @@ -369,7 +373,7 @@ describe('automatic host preparation', () => { it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => { const version = await publish(); - const id = enroll({ runtimeImages: ['another:runtime'] }); + const id = enroll({ preparedRuntimeImages: ['another:runtime'] }); const host = hosts.getHost(id)!; const status = preparation.hostPreparationStatus(host); expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']); @@ -406,7 +410,7 @@ describe('automatic host preparation', () => { preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); - heartbeat(id, { runtimeImages: [runtime] }); + heartbeat(id, { preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(2); const download = operations(id)[0]; @@ -417,7 +421,7 @@ describe('automatic host preparation', () => { verified(id, version); hosts.reportHostOperation(id, download.id, { state: 'completed' }); heartbeat(id, { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); @@ -432,7 +436,7 @@ describe('automatic host preparation', () => { game: 't5', files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }] }); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); const expected = new Map([latest, t6].map((version) => [version.id, version])); const downloaded: string[] = []; for (let index = 0; index < expected.size; index++) { @@ -452,7 +456,7 @@ describe('automatic host preparation', () => { preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(index + 1); heartbeat(id, { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: downloaded.map((id) => ({ id, state: 'ready' })) }); } @@ -462,12 +466,15 @@ describe('automatic host preparation', () => { }); it('picks up publication after enrollment and keeps verified historical versions intact', async () => { - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); expect(operations(id)).toEqual([]); const old = await publish(); verified(id, old); - heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] }); + heartbeat(id, { + preparedRuntimeImages: [runtime], + libraries: [{ id: old.id, state: 'ready' }] + }); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); const latest = await publish(); preparation.reconcileHostPreparation(id); @@ -479,7 +486,7 @@ describe('automatic host preparation', () => { it('replaces an undelivered automatic download when newer files finish uploading', async () => { const old = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const first = operations(id)[0]; expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true }); @@ -503,8 +510,8 @@ describe('automatic host preparation', () => { async (automatic) => { const old = await publish(); const image = 'ghcr.io/example/plutainer:queued-version'; - const id = enroll({ runtimeImages: [runtime, image] }); - const other = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime, image] }); + const other = enroll({ preparedRuntimeImages: [runtime] }); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); for (let index = 0; index < 2; index++) { @@ -521,7 +528,7 @@ describe('automatic host preparation', () => { ).toBe(true); } // Also cover pre-existing intents whose image is no longer cached. - heartbeat(id, { runtimeImages: [runtime] }); + heartbeat(id, { preparedRuntimeImages: [runtime] }); const latest = await publish(); const status = preparation.hostPreparationStatus(hosts.getHost(id)!); expect(status.items.filter((item) => item.label === image)).toHaveLength(0); @@ -555,7 +562,7 @@ describe('automatic host preparation', () => { vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, download.id, { state: 'completed' }); heartbeat(id, { - runtimeImages: [runtime, image], + preparedRuntimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] }); preparation.reconcileHostPreparation(id); @@ -568,7 +575,7 @@ describe('automatic host preparation', () => { it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => { const old = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); @@ -604,7 +611,7 @@ describe('automatic host preparation', () => { it('finishes a delivered download before following a newer upload under a new operation ID', async () => { const old = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const [first] = hosts.pendingHostOperations(id); const latest = await publish(); @@ -613,7 +620,10 @@ describe('automatic host preparation', () => { expect(hosts.getHostOperation(first.id)?.state).toBe('pending'); verified(id, old); hosts.reportHostOperation(id, first.id, { state: 'completed' }); - heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] }); + heartbeat(id, { + preparedRuntimeImages: [runtime], + libraries: [{ id: old.id, state: 'ready' }] + }); preparation.reconcileHostPreparation(id); expect(operations(id)[0]).toMatchObject({ state: 'pending', @@ -688,23 +698,26 @@ describe('automatic host preparation', () => { it('blocks a library that cannot fit while preserving the host free-space reserve', async () => { const version = await publish(); const id = enroll({ - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(operations(id)).toEqual([]); - heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes }); + heartbeat(id, { + preparedRuntimeImages: [runtime], + storageAvailable: 1_000 + version.sizeBytes + }); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); expect(operations(id)[0].payload.libraryVersionId).toBe(version.id); }); it('returns an active download even when its in-progress bytes have consumed free space', async () => { const version = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); const operation = libraries.queueLibraryDownload(id, version.id); heartbeat(id, { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes }); expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id); @@ -715,7 +728,7 @@ describe('automatic host preparation', () => { it('prepares T6 when the latest T5 upload cannot fit', async () => { const oversized = await publish(); const small = await publish('t6', 'small'); - const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 }); + const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); const status = preparation.reconcileHostPreparation(id); expect(status.items).toEqual( @@ -731,7 +744,7 @@ describe('automatic host preparation', () => { it('does not count partial bytes from a different manifest toward download storage', async () => { const version = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); libraries.updateHostLibraryCaches(id, [ { libraryVersionId: version.id, @@ -741,7 +754,7 @@ describe('automatic host preparation', () => { } ]); heartbeat(id, { - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); @@ -753,7 +766,7 @@ describe('automatic host preparation', () => { it('resumes a failed verified-manifest download using only its remaining storage', async () => { const version = await publish(); - const id = enroll({ runtimeImages: [runtime] }); + const id = enroll({ preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const operation = operations(id)[0]; libraries.updateHostLibraryCaches(id, [ @@ -768,11 +781,11 @@ describe('automatic host preparation', () => { hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' }); const failedAt = Date.now(); const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80; - heartbeat(id, { runtimeImages: [runtime], storageAvailable: free }); + heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); vi.setSystemTime(failedAt + 60_000); - heartbeat(id, { runtimeImages: [runtime], storageAvailable: free }); + heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free }); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); expect(operations(id)).toHaveLength(2); expect(operations(id)[0].payload.libraryVersionId).toBe(version.id); @@ -800,7 +813,7 @@ describe('optional host Plutonium cache', () => { (state) => { const ready = { ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] }; const id = enroll(ready); @@ -819,7 +832,7 @@ describe('optional host Plutonium cache', () => { it('delivers newer deployments before queued cache work and before a cache replay', () => { const id = enroll({ ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' }] }); preparation.reconcileHostPreparation(id); @@ -842,7 +855,7 @@ describe('optional host Plutonium cache', () => { const ready = { ...inventory, ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] }; const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready); @@ -903,7 +916,7 @@ describe('optional host Plutonium cache', () => { const version = await publish(); const ready = { ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }], plutoniumCache: { revision: 5354, bytes: 200 } }; @@ -948,7 +961,7 @@ describe('optional host Plutonium cache', () => { const pull = operations(id)[0]; expect(pull.kind).toBe('runtime.pull'); hosts.reportHostOperation(id, pull.id, { state: 'completed' }); - heartbeat(id, { ...capable, runtimeImages: [runtime] }); + heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; expect(download.kind).toBe('library.download'); @@ -956,7 +969,7 @@ describe('optional host Plutonium cache', () => { hosts.reportHostOperation(id, download.id, { state: 'completed' }); const ready = { ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] }; heartbeat(id, ready); @@ -1005,7 +1018,7 @@ describe('optional host Plutonium cache', () => { it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => { const ready = { ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 25_000 } }; const id = enroll(ready); @@ -1048,7 +1061,7 @@ describe('optional host Plutonium cache', () => { checkedAt: new Date(Date.now() - 60 * 60_000).toISOString() }); if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' }); - const id = enroll({ ...capable, runtimeImages: [runtime] }); + const id = enroll({ ...capable, preparedRuntimeImages: [runtime] }); preparation.reconcileHostPreparation(id); expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', @@ -1060,7 +1073,7 @@ describe('optional host Plutonium cache', () => { ).toBe('Latest production revision'); const cached = enroll({ ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], plutoniumCache: { revision: 5353, bytes: 200 } }); preparation.reconcileHostPreparation(cached); @@ -1071,7 +1084,7 @@ describe('optional host Plutonium cache', () => { it('leaves agents without the cache capability unchanged', async () => { await release(5355); const id = enroll({ - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 200 } }); expect( @@ -1084,7 +1097,7 @@ describe('optional host Plutonium cache', () => { const version = await publish(); const ready = { ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] }; const id = enroll(ready); @@ -1135,7 +1148,7 @@ describe('optional host Plutonium cache', () => { const version = await publish(); const id = enroll({ ...capable, - runtimeImages: [runtime], + preparedRuntimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); diff --git a/tests/unit/host-request-cost.test.ts b/tests/unit/host-request-cost.test.ts index 6b6d907..eb6a8b7 100644 --- a/tests/unit/host-request-cost.test.ts +++ b/tests/unit/host-request-cost.test.ts @@ -11,7 +11,7 @@ const inventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['plutainer:test'], + preparedRuntimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }] }; beforeEach(async () => { diff --git a/tests/unit/native-creation-intents.test.ts b/tests/unit/native-creation-intents.test.ts index 9a18e04..8c9b0ec 100644 --- a/tests/unit/native-creation-intents.test.ts +++ b/tests/unit/native-creation-intents.test.ts @@ -193,7 +193,7 @@ describe('native creation intents before placement', () => { memoryTotal: 32 * 1024 ** 3, memoryAvailable: 30 * 1024 ** 3, storageAvailable: 100_000_000_000, - runtimeImages: [image] + preparedRuntimeImages: [image] }; hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory); const libraries = await import('../../src/lib/server/game-files/index.js'); diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts index 1713e53..0dfd3d0 100644 --- a/tests/unit/native-hosts.test.ts +++ b/tests/unit/native-hosts.test.ts @@ -12,7 +12,7 @@ const inventory = { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, - runtimeImages: ['plutainer:test'], + preparedRuntimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }] }; beforeEach(async () => { @@ -494,6 +494,21 @@ describe('native host enrollment and scoped authentication', () => { }); describe('native host readiness and reservations', () => { + it.each([ + { preparedRuntimeImages: 'plutainer:test' }, + { preparedRuntimeImages: [123] }, + { preparedRuntimeImages: null } + ])( + 'rejects malformed prepared runtime inventory: $preparedRuntimeImages', + async ({ preparedRuntimeImages }) => { + const api = await enrolled(); + expect(() => + api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages }) + ).toThrow(/runtime image inventory/i); + expect(api.requireReadyHost(api.host.id).readiness).toBe('ready'); + } + ); + it('separates a connected agent from ready Docker, storage, runtime and library state', async () => { const api = await enrolled(); expect(api.requireReadyHost(api.host.id).readiness).toBe('ready'); @@ -518,8 +533,8 @@ describe('native host readiness and reservations', () => { }); it.each([ { missing: { libraries: [] }, expected: 'needs-game-files' }, - { missing: { runtimeImages: [] }, expected: 'needs-runtime' }, - { missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' } + { missing: { preparedRuntimeImages: [] }, expected: 'needs-runtime' }, + { missing: { preparedRuntimeImages: [], libraries: [] }, expected: 'needs-runtime' } ])( 'names missing content as $expected until a prepared heartbeat arrives', async ({ missing, expected }) => { @@ -545,7 +560,7 @@ describe('native host readiness and reservations', () => { download.id, runtime.id ]); - api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] }); + api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages: [], libraries: [] }); expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([ download.id, runtime.id diff --git a/tests/unit/native-launch-preparation.test.ts b/tests/unit/native-launch-preparation.test.ts index b09e2ee..29c814a 100644 --- a/tests/unit/native-launch-preparation.test.ts +++ b/tests/unit/native-launch-preparation.test.ts @@ -39,7 +39,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, - runtimeImages: [image] + preparedRuntimeImages: [image] }; enrollHost(createEnrollment(hostId).token, inventory); const { createLibraryImport, updateHostLibraryCaches } = diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index e35875d..2185976 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -43,7 +43,7 @@ const inventory = () => ({ dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, - runtimeImages: [image], + preparedRuntimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] }); beforeEach(async () => { diff --git a/tests/unit/native-safety.test.ts b/tests/unit/native-safety.test.ts index b1112a5..2f3218d 100644 --- a/tests/unit/native-safety.test.ts +++ b/tests/unit/native-safety.test.ts @@ -34,7 +34,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, - runtimeImages: ['plutainer:test'], + preparedRuntimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }], capabilities: ['instance.safety-override'] }); diff --git a/tests/unit/rebalancing-runs.test.ts b/tests/unit/rebalancing-runs.test.ts index 9bc2bcc..c76f2b0 100644 --- a/tests/unit/rebalancing-runs.test.ts +++ b/tests/unit/rebalancing-runs.test.ts @@ -47,7 +47,7 @@ async function heartbeat(hostId: string, cpuUsagePercent: number) { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * gib, - runtimeImages: [image], + preparedRuntimeImages: [image], libraries: [{ id: libraryId, state: 'ready' }], instanceResources: instances.map((instance) => ({ instanceId: instance.id, @@ -92,7 +92,7 @@ beforeEach(async () => { dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * gib, - runtimeImages: [image], + preparedRuntimeImages: [image], libraries: [{ id: libraryId, state: 'ready' }] }); const libraries = await import('../../src/lib/server/game-files/index.js'); diff --git a/tests/unit/retention.test.ts b/tests/unit/retention.test.ts index 2733cb2..09cf2cf 100644 --- a/tests/unit/retention.test.ts +++ b/tests/unit/retention.test.ts @@ -548,7 +548,7 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') { dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, - runtimeImages: [config.defaultRuntimeImage], + preparedRuntimeImages: [config.defaultRuntimeImage], libraries: [{ id: 'lib', state: 'ready' }] }); const generation = action === 'create' ? 1 : 2; diff --git a/tests/unit/runtime-image-policy.test.ts b/tests/unit/runtime-image-policy.test.ts index f9e80f9..3fb4ae0 100644 --- a/tests/unit/runtime-image-policy.test.ts +++ b/tests/unit/runtime-image-policy.test.ts @@ -23,16 +23,31 @@ afterEach(async () => { await fixture.cleanup(); }); -async function cache(images: string[]) { +async function reportImages( + preparedImages: string[] | undefined, + cachedImages = preparedImages ?? [] +) { const hosts = await import('../../src/lib/server/hosts/index.js'); const host = hosts.getHost(prepared.host.id)!; - hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images }); + hosts.heartbeat(host.id, { + ...host.inventory, + runtimeImages: cachedImages, + preparedRuntimeImages: preparedImages + }); } describe('runtime image launch policy', () => { - it.each([false, true])( - 'rejects an uncached custom image at POST /api/servers (automatic: %s)', - async (automatic) => { + it.each([ + { automatic: false, cached: false, legacy: false }, + { automatic: true, cached: false, legacy: false }, + { automatic: false, cached: true, legacy: false }, + { automatic: true, cached: true, legacy: false }, + { automatic: false, cached: true, legacy: true }, + { automatic: true, cached: true, legacy: true } + ])( + 'rejects an unprepared custom image at POST /api/servers ($automatic, cached: $cached, legacy: $legacy)', + async ({ automatic, cached, legacy }) => { + await reportImages(legacy ? undefined : [], cached ? [custom] : []); const { POST } = await import('../../src/routes/api/servers/+server.js'); const response = await POST({ request: new Request('https://controller.test/api/servers', { @@ -57,13 +72,88 @@ describe('runtime image launch policy', () => { ); it('allows the configured default before it is cached', async () => { - await cache([]); + await reportImages([]); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); expect(queueNativeCreation(launch).ok).toBe(true); }); + it('keeps the cached configured default eligible with a legacy agent', async () => { + const { config } = await import('../../src/lib/server/config.js'); + await reportImages(undefined, [config.defaultRuntimeImage, custom]); + const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); + expect(nativeSelection({}, 't5').hostId).toBe(prepared.host.id); + const { getHost } = await import('../../src/lib/server/hosts/index.js'); + expect(getHost(prepared.host.id)!.inventory.preparedRuntimeImages).toEqual([ + config.defaultRuntimeImage + ]); + }); + + it('ignores malformed preparation fields already stored by an older controller', async () => { + const { config } = await import('../../src/lib/server/config.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { getHost } = await import('../../src/lib/server/hosts/index.js'); + const inventory = getHost(prepared.host.id)!.inventory; + for (const reported of [{}, 42, custom]) { + db() + .prepare('UPDATE ec_hosts SET inventory=? WHERE id=?') + .run( + JSON.stringify({ + ...inventory, + runtimeImages: [config.defaultRuntimeImage, custom], + preparedRuntimeImages: reported + }), + prepared.host.id + ); + expect(getHost(prepared.host.id)).toMatchObject({ + readiness: 'ready', + inventory: { preparedRuntimeImages: [config.defaultRuntimeImage] } + }); + } + }); + + it('hides unrelated cached images from host list, detail, and page responses', async () => { + const { config } = await import('../../src/lib/server/config.js'); + await reportImages(undefined, [config.defaultRuntimeImage, custom]); + const list = await import('../../src/routes/api/hosts/+server.js'); + const detail = await import('../../src/routes/api/hosts/[id]/+server.js'); + const { load } = await import('../../src/routes/hosts/+page.server.js'); + const request = new Request('https://controller.test/api/hosts'); + const bodies = [ + await list.GET({ request }).json(), + await detail.GET({ request, params: { id: prepared.host.id } }).json(), + load() + ]; + for (const body of bodies) { + const host = 'hosts' in body ? body.hosts[0] : body.host; + expect(host.inventory.runtimeImages).toBeUndefined(); + expect(host.inventory.preparedRuntimeImages).toEqual([config.defaultRuntimeImage]); + expect(JSON.stringify(body)).not.toContain(custom); + } + }); + + it('does not place an accepted runtime on another host that only caches it', async () => { + await reportImages([custom]); + const other = await prepareNativeHost(); + const hosts = await import('../../src/lib/server/hosts/index.js'); + hosts.heartbeat(other.host.id, { + ...hosts.getHost(other.host.id)!.inventory, + runtimeImages: [custom] + }); + expect(() => hosts.requireReadyHost(other.host.id, { image: custom })).toThrow( + /runtime image is not prepared/ + ); + const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js'); + expect(() => + choosePlacement({ + hostId: other.host.id, + libraryId: other.versions.get('t5')!, + image: custom + }) + ).toThrow(/runtime image is not prepared/); + }); + it('accepts prepared images only on hosts that have them', async () => { - await cache([custom]); + await reportImages([custom]); const hosts = await import('../../src/lib/server/hosts/index.js'); const other = hosts.createHost({ name: 'Unprepared host' }); const { nativeLaunchSelection, nativeSelection } = @@ -81,7 +171,7 @@ describe('runtime image launch policy', () => { ); }); - it('requires exact image references and does not treat a queued admin pull as cached', async () => { + it('requires exact prepared references and does not treat a queued admin pull as preparation', async () => { const hosts = await import('../../src/lib/server/hosts/index.js'); hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom }); const { nativeLaunchSelection } = @@ -89,25 +179,25 @@ describe('runtime image launch policy', () => { expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow( /prepared by an admin/ ); - await cache([custom + '-other']); + await reportImages([custom + '-other']); expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow( /prepared by an admin/ ); }); - it('rechecks cached images when submitting a saved single-server preview', async () => { - await cache([custom]); + it('rechecks preparation when submitting a saved preview even if the image is still cached', async () => { + await reportImages([custom]); const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const preview = previewNativeCreation({ ...launch, image: custom }); - await cache([]); + await reportImages([], [custom]); expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({ ok: false, message: expect.stringContaining('prepared by an admin') }); }); - it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => { + it('rejects unprepared collection images and rechecks saved previews before writing any jobs', async () => { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); @@ -115,15 +205,16 @@ describe('runtime image launch policy', () => { name: 'Image policy', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] }); + await reportImages([], [custom]); expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow( /prepared by an admin/ ); - await cache([custom]); + await reportImages([custom]); const preview = previewCollection(blueprint.id, 'fill', { image: custom }); const keys = Object.fromEntries( preview.servers.map((server) => [server.name!, 'key-' + server.name]) ); - await cache([]); + await reportImages([], [custom]); expect(() => queueCollection({ previewId: preview.id, keys })).toThrow( /prepared by an admin/ ); @@ -136,7 +227,7 @@ describe('runtime image launch policy', () => { ]) { expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); } - await cache([custom]); + await reportImages([custom]); expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true); }); });