This commit is contained in:
+10
-4
@@ -35,7 +35,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.8.5"
|
||||
VERSION = "1.8.6"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -1462,7 +1462,7 @@ class Agent:
|
||||
progress = list(progress.values())
|
||||
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
||||
caches.extend(progress)
|
||||
images = []
|
||||
images, prepared_images = [], []
|
||||
if docker_running:
|
||||
with contextlib.suppress(AgentError):
|
||||
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
||||
@@ -1480,7 +1480,13 @@ class Agent:
|
||||
aliases = self.root / "agent" / "runtime-images.json"
|
||||
if aliases.exists():
|
||||
with contextlib.suppress(OSError, ValueError):
|
||||
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
|
||||
saved = json.loads(aliases.read_text())
|
||||
if isinstance(saved, dict):
|
||||
# Only runtime.pull records administrator-prepared
|
||||
# references. Other Docker cache entries are useful
|
||||
# diagnostics, but do not grant launch permission.
|
||||
prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids]
|
||||
images.extend(prepared_images)
|
||||
resources = {}
|
||||
if docker_running:
|
||||
observations = []
|
||||
@@ -1499,7 +1505,7 @@ class Agent:
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
|
||||
@@ -2502,12 +2502,62 @@ class AgentTests(unittest.TestCase):
|
||||
return f"ghcr.io/ayymoss/plutainer|latest|{image_digest}|{image_id}\n"
|
||||
return original(*args, **kwargs)
|
||||
self.docker.call = call
|
||||
module.atomic_json(self.agent.root / "agent/runtime-images.json", {"ghcr.io/ayymoss/plutainer": image_id, "removed-image": "missing-id"})
|
||||
images = self.agent.inventory()["runtimeImages"]
|
||||
prepared = ["ghcr.io/ayymoss/plutainer", "ghcr.io/ayymoss/plutainer@" + image_digest]
|
||||
module.atomic_json(self.agent.root / "agent/runtime-images.json", {**dict.fromkeys(prepared, image_id), "removed-image": "missing-id"})
|
||||
inventory = self.agent.inventory()
|
||||
images = inventory["runtimeImages"]
|
||||
self.assertIn("ghcr.io/ayymoss/plutainer@" + image_digest, images)
|
||||
self.assertIn("ghcr.io/ayymoss/plutainer:latest", images)
|
||||
self.assertIn("ghcr.io/ayymoss/plutainer", images)
|
||||
self.assertNotIn("removed-image", images)
|
||||
self.assertEqual(inventory["preparedRuntimeImages"], prepared)
|
||||
|
||||
def test_runtime_pull_prepares_only_the_requested_reference_from_the_docker_cache(self):
|
||||
image = "registry.example/custom-runtime:stable"
|
||||
image_id = "sha256:" + "1" * 64
|
||||
original = self.docker.call
|
||||
|
||||
def call(*args, **kwargs):
|
||||
if args[:2] == ("image", "ls"):
|
||||
return (f"registry.example/custom-runtime|stable|<none>|{image_id}\n"
|
||||
f"registry.example/custom-runtime|unprepared-alias|<none>|{image_id}\n"
|
||||
"unrelated-service|latest|<none>|sha256:unrelated\n")
|
||||
if args[:2] == ("image", "inspect"):
|
||||
self.assertEqual(args[-1], image)
|
||||
return image_id
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.docker.call = call
|
||||
before = self.agent.inventory()
|
||||
self.assertIn(image, before["runtimeImages"])
|
||||
self.assertIn("unrelated-service:latest", before["runtimeImages"])
|
||||
self.assertEqual(before["preparedRuntimeImages"], [])
|
||||
|
||||
result = self.agent.execute(self.operation("runtime.pull", {"image": image}))
|
||||
self.assertEqual(result["state"], "completed")
|
||||
inventory = self.agent.inventory()
|
||||
self.assertEqual(inventory["runtimeImages"], before["runtimeImages"])
|
||||
self.assertEqual(inventory["preparedRuntimeImages"], [image])
|
||||
|
||||
def test_runtime_preparation_inventory_fails_closed_when_records_or_docker_are_unavailable(self):
|
||||
aliases = self.agent.root / "agent/runtime-images.json"
|
||||
original = self.docker.call
|
||||
|
||||
def call(*args, **kwargs):
|
||||
if args[:2] == ("image", "ls"):
|
||||
return "custom-runtime|latest|<none>|sha256:prepared\n"
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.docker.call = call
|
||||
for content in ("invalid json", "[]", '{"custom-runtime:latest": []}', '{"custom-runtime:latest": "sha256:removed"}'):
|
||||
with self.subTest(content=content):
|
||||
aliases.write_text(content)
|
||||
inventory = self.agent.inventory()
|
||||
self.assertEqual(inventory["preparedRuntimeImages"], [])
|
||||
self.assertEqual(inventory["runtimeImages"], ["custom-runtime:latest"])
|
||||
module.atomic_json(aliases, {"custom-runtime:latest": "sha256:prepared"})
|
||||
with patch.object(self.docker, "call", side_effect=AgentError("Docker unavailable")):
|
||||
self.assertEqual(self.agent.inventory()["preparedRuntimeImages"], [])
|
||||
|
||||
def test_heartbeat_applies_configured_host_port_range_before_deployment(self):
|
||||
self.agent.request = lambda *_: {"host": {"portMin": 30000, "portMax": 30100}}
|
||||
|
||||
@@ -243,10 +243,15 @@ servers. Existing API clients may still explicitly constrain a launch to a host.
|
||||
Queued jobs retain their accepted game-file version. Automatic host preparation
|
||||
also downloads the game files needed by unplaced jobs; a newer upload does not
|
||||
remove files still needed by the queue. Only the configured default runtime is
|
||||
pulled automatically. Custom images must already be cached on the selected host,
|
||||
or on an active host for automatic placement, before a launch is accepted. Admins
|
||||
can prepare them from the host page. Queued launches and migrations never trigger
|
||||
custom image pulls, and container creation cannot implicitly pull a missing image.
|
||||
pulled automatically. Custom images must have been prepared by an admin from the
|
||||
selected host's page, or on an active host for automatic placement, before a launch
|
||||
is accepted. An image cached by another Docker workload does not qualify. Agent
|
||||
1.8.6 reports these exact prepared references from `runtime-images.json` while
|
||||
their image IDs remain cached; the controller also trusts its configured default
|
||||
when cached. Host pages and APIs expose only prepared runtimes. Older agents can
|
||||
still run the configured default, but must update before accepting custom runtimes.
|
||||
Queued launches and migrations never trigger custom image pulls, and container
|
||||
creation cannot implicitly pull a missing image.
|
||||
|
||||
### Resource samples and learned allowances
|
||||
|
||||
|
||||
@@ -163,7 +163,7 @@ export function seedFixtureData(handle: Database.Database, now: number): void {
|
||||
storageAvailable: 240 * GIB,
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [
|
||||
{ id: 'fixture-t5', state: 'ready' },
|
||||
{ id: 'fixture-t6', state: 'ready' }
|
||||
|
||||
@@ -156,7 +156,10 @@ export interface HostInventory {
|
||||
dockerVersion?: string;
|
||||
dockerRunning?: boolean;
|
||||
directoriesWritable?: boolean;
|
||||
/** Agent-reported Docker cache; removed from presented host inventory. */
|
||||
runtimeImages?: string[];
|
||||
/** Prepared, locally available image references eligible for launches. */
|
||||
preparedRuntimeImages?: string[];
|
||||
libraries?: HostLibrary[];
|
||||
currentOperation?: string | null;
|
||||
[key: string]: unknown;
|
||||
|
||||
@@ -198,7 +198,7 @@ export function launchAvailability(selections: NativePlacement[]): {
|
||||
);
|
||||
const eligible = (candidate: HostAvailability, selection: NativePlacement) =>
|
||||
(!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) &&
|
||||
!!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') &&
|
||||
!!candidate.host.inventory.preparedRuntimeImages?.includes(selection.image ?? '') &&
|
||||
!!candidate.host.inventory.libraries?.some(
|
||||
(library) =>
|
||||
library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)
|
||||
|
||||
@@ -66,7 +66,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
|
||||
if (!host.credentialActive && host.enrolled)
|
||||
reasons.push('Host credentials are inactive.');
|
||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage))
|
||||
if (!host.inventory.preparedRuntimeImages?.includes(config.defaultRuntimeImage))
|
||||
reasons.push('The default runtime is not ready.');
|
||||
if (!requestedGames.some((game) => games.includes(game)))
|
||||
reasons.push('Latest game files for this selection are not ready.');
|
||||
|
||||
@@ -140,7 +140,7 @@ function hostFromRow(row: HostRow): Host {
|
||||
)
|
||||
reasons.push('Insufficient available game storage.');
|
||||
const infrastructureReady = reasons.length === 0;
|
||||
const runtimeReady = !!inventory.runtimeImages?.length;
|
||||
const runtimeReady = !!inventory.preparedRuntimeImages?.length;
|
||||
const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready');
|
||||
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
|
||||
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
|
||||
@@ -676,12 +676,14 @@ export function validateInventory(value: unknown): HostInventory {
|
||||
for (const key of ['dockerRunning', 'directoriesWritable'])
|
||||
if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean')
|
||||
throw new Error(`Invalid inventory ${key}.`);
|
||||
if (
|
||||
inventory.runtimeImages !== undefined &&
|
||||
(!Array.isArray(inventory.runtimeImages) ||
|
||||
inventory.runtimeImages.some((image) => typeof image !== 'string'))
|
||||
)
|
||||
throw new Error('Invalid runtime image inventory.');
|
||||
for (const key of ['runtimeImages', 'preparedRuntimeImages'] as const) {
|
||||
const images = inventory[key];
|
||||
if (
|
||||
images !== undefined &&
|
||||
(!Array.isArray(images) || images.some((image) => typeof image !== 'string'))
|
||||
)
|
||||
throw new Error('Invalid runtime image inventory.');
|
||||
}
|
||||
if (
|
||||
inventory.capabilities !== undefined &&
|
||||
(!Array.isArray(inventory.capabilities) ||
|
||||
@@ -952,8 +954,8 @@ export function requireReadyHost(
|
||||
)
|
||||
)
|
||||
reasons.push('The selected game library is not ready on this host.');
|
||||
if (options.image && !host.inventory.runtimeImages?.includes(options.image))
|
||||
reasons.push('The selected runtime image is not cached on this host.');
|
||||
if (options.image && !host.inventory.preparedRuntimeImages?.includes(options.image))
|
||||
reasons.push('The selected runtime image is not prepared on this host.');
|
||||
if (
|
||||
options.port !== undefined &&
|
||||
!hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)
|
||||
|
||||
@@ -190,7 +190,7 @@ function targets(host: Host): Target[] {
|
||||
const result: Target[] = [config.defaultRuntimeImage].map((image) => {
|
||||
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
||||
const runtimeReady =
|
||||
host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||
host.inventory.preparedRuntimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||
return {
|
||||
image,
|
||||
requiredBytes: host.minimumFreeBytes,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { config } from '../config.js';
|
||||
import type { HostInventory } from '../../hosts.js';
|
||||
|
||||
type InventoryCache = Map<string, { json: string; inventory: HostInventory }>;
|
||||
@@ -21,7 +22,20 @@ export function parseHostInventory(hostId: string, json: string): HostInventory
|
||||
const cache = requests.getStore()?.inventories;
|
||||
const saved = cache?.get(hostId);
|
||||
if (saved?.json === json) return saved.inventory;
|
||||
const inventory: HostInventory = JSON.parse(json);
|
||||
// Full Docker inventory can include unrelated workloads. Keep it out of
|
||||
// every presented host, including inventory persisted by older agents.
|
||||
const { runtimeImages, ...inventory }: HostInventory = JSON.parse(json);
|
||||
// The configured default is explicitly trusted by the controller. Older
|
||||
// agents may prove it is cached, but cannot authorize any custom runtime.
|
||||
const reported = inventory.preparedRuntimeImages;
|
||||
const prepared = new Set(
|
||||
Array.isArray(reported) && reported.every((image) => typeof image === 'string')
|
||||
? reported
|
||||
: []
|
||||
);
|
||||
if (runtimeImages?.includes(config.defaultRuntimeImage))
|
||||
prepared.add(config.defaultRuntimeImage);
|
||||
inventory.preparedRuntimeImages = [...prepared];
|
||||
// Compare the stored JSON so heartbeats and transaction rollbacks within
|
||||
// a request remain visible. Other host state is always read afresh.
|
||||
cache?.set(hostId, { json, inventory });
|
||||
|
||||
@@ -136,7 +136,7 @@ export function nativeLaunchSelection(
|
||||
throw new Error('The selected host is being removed. Choose another host.');
|
||||
}
|
||||
// Only the controller-configured default may be prepared automatically.
|
||||
// Custom runtimes must already be present in trusted host inventory; a
|
||||
// Custom runtimes must be reported as admin-prepared by the host agent; a
|
||||
// launch request must never authorize a registry download.
|
||||
if (
|
||||
image !== config.defaultRuntimeImage &&
|
||||
@@ -144,7 +144,7 @@ export function nativeLaunchSelection(
|
||||
(host) =>
|
||||
!host.decommission &&
|
||||
(automatic ? host.mode === 'active' : host.id === input.hostId) &&
|
||||
host.inventory.runtimeImages?.includes(image)
|
||||
host.inventory.preparedRuntimeImages?.includes(image)
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
hosts.filter(
|
||||
(host) =>
|
||||
host.readiness === 'ready' &&
|
||||
host.inventory.runtimeImages?.includes(selection.image)
|
||||
host.inventory.preparedRuntimeImages?.includes(selection.image)
|
||||
)
|
||||
);
|
||||
const latest = $derived(latestGameFiles(libraries));
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function GET() {
|
||||
cache.status === 'ready' && readyHosts.some((host) => host.id === cache.hostId)
|
||||
)
|
||||
);
|
||||
const runtimes = connected.filter((host) => host.inventory.runtimeImages?.length);
|
||||
const runtimes = connected.filter((host) => host.inventory.preparedRuntimeImages?.length);
|
||||
const checks: SetupCheck[] = [
|
||||
{
|
||||
id: 'hosts',
|
||||
@@ -46,7 +46,7 @@ export async function GET() {
|
||||
title: 'Plutainer runtime',
|
||||
state: runtimes.length ? 'ready' : 'attention',
|
||||
detail: runtimes.length
|
||||
? `${runtimes.length} connected hosts report cached runtime images.`
|
||||
? `${runtimes.length} connected hosts report prepared runtime images.`
|
||||
: 'Hosts prepare the configured Plutainer runtime automatically.',
|
||||
href: '/hosts',
|
||||
action: 'View host setup'
|
||||
|
||||
@@ -447,7 +447,7 @@
|
||||
</li>{/each}
|
||||
</ul>{:else}<p>No cached game libraries reported yet.</p>{/if}
|
||||
<p>
|
||||
Cached images: {host.inventory.runtimeImages?.join(', ') ||
|
||||
Prepared runtimes: {host.inventory.preparedRuntimeImages?.join(', ') ||
|
||||
'None reported yet'}
|
||||
</p>
|
||||
{#if canManage}<details>
|
||||
|
||||
@@ -35,7 +35,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.8.5"
|
||||
VERSION = "1.8.6"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
MANIFEST_MAX_ENTRIES = 10000
|
||||
@@ -1462,7 +1462,7 @@ class Agent:
|
||||
progress = list(progress.values())
|
||||
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
||||
caches.extend(progress)
|
||||
images = []
|
||||
images, prepared_images = [], []
|
||||
if docker_running:
|
||||
with contextlib.suppress(AgentError):
|
||||
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
||||
@@ -1480,7 +1480,13 @@ class Agent:
|
||||
aliases = self.root / "agent" / "runtime-images.json"
|
||||
if aliases.exists():
|
||||
with contextlib.suppress(OSError, ValueError):
|
||||
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
|
||||
saved = json.loads(aliases.read_text())
|
||||
if isinstance(saved, dict):
|
||||
# Only runtime.pull records administrator-prepared
|
||||
# references. Other Docker cache entries are useful
|
||||
# diagnostics, but do not grant launch permission.
|
||||
prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids]
|
||||
images.extend(prepared_images)
|
||||
resources = {}
|
||||
if docker_running:
|
||||
observations = []
|
||||
@@ -1499,7 +1505,7 @@ class Agent:
|
||||
update = self.update_inventory()
|
||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
plutonium_cache = self.plutonium_cache_inventory()
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||
health = self.inspect(instance_id, generation)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"version": "1.8.5",
|
||||
"version": "1.8.6",
|
||||
"protocolVersion": 1,
|
||||
"platform": "debian-amd64",
|
||||
"sha256": "56187a244b551af884db6bd6419daf7dd7d931fa1e5d5c5414559bb6e25f1094",
|
||||
"sha256": "79702dc8cf80d2521153bfda9241f7240285014fe746fc921f0a7793dd81212b",
|
||||
"updaterProtocolVersion": 1,
|
||||
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ export async function prepareNativeHost(servers: string[] = []) {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [config.defaultRuntimeImage]
|
||||
preparedRuntimeImages: [config.defaultRuntimeImage]
|
||||
};
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
||||
const versions = new Map<string, string>();
|
||||
|
||||
@@ -10,7 +10,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }],
|
||||
capabilities: ['instance.cancel-create', 'operations.health-lane']
|
||||
};
|
||||
|
||||
@@ -18,7 +18,7 @@ const inventory = {
|
||||
cpuUsagePercent: 0,
|
||||
memoryTotal: 64 * GiB,
|
||||
memoryAvailable: 60 * GiB,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [
|
||||
{ id: 't5-version', state: 'ready' },
|
||||
{ id: 't6-version', state: 'ready' }
|
||||
|
||||
@@ -35,7 +35,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
enrollHost(createEnrollment(hostId).token, inventory);
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -47,7 +47,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -33,7 +33,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: [runtime]
|
||||
preparedRuntimeImages: [runtime]
|
||||
});
|
||||
version = await publish();
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
const lane: HostAgentLane = {
|
||||
|
||||
@@ -15,7 +15,7 @@ const inventory = {
|
||||
cpuUsagePercent: 0,
|
||||
memoryTotal: 64 * GiB,
|
||||
memoryAvailable: 60 * GiB,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [
|
||||
{ id: 't5-library', state: 'ready' },
|
||||
{ id: 't6-library', state: 'ready' }
|
||||
@@ -46,6 +46,32 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe('cached launch availability', () => {
|
||||
it.each([{ preparedRuntimeImages: [] }, { preparedRuntimeImages: ['another:image'] }])(
|
||||
'excludes a cached image until it is prepared ($preparedRuntimeImages)',
|
||||
async ({ preparedRuntimeImages }) => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { launchAvailability } =
|
||||
await import('../../src/lib/server/hosts/availability.js');
|
||||
heartbeat(hostId, {
|
||||
...inventory,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages
|
||||
});
|
||||
const unavailable = launchAvailability([t5]);
|
||||
expect(unavailable.availability).toMatchObject({
|
||||
estimatedAvailable: 0,
|
||||
portsAvailable: 0
|
||||
});
|
||||
expect(unavailable.warnings.join(' ')).toContain('Waiting for an eligible host');
|
||||
|
||||
heartbeat(hostId, { ...inventory, runtimeImages: ['image'] });
|
||||
expect(launchAvailability([t5]).availability).toMatchObject({
|
||||
estimatedAvailable: 1,
|
||||
portsAvailable: 3
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
||||
|
||||
@@ -14,7 +14,7 @@ const inventory = {
|
||||
protocolVersion: 1,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * GiB,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
const sample = {
|
||||
|
||||
@@ -9,7 +9,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
let fixture: Fixture;
|
||||
|
||||
@@ -18,7 +18,7 @@ const inventory: HostInventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: [],
|
||||
preparedRuntimeImages: [],
|
||||
libraries: [],
|
||||
capabilities: ['host.decommission']
|
||||
};
|
||||
@@ -123,7 +123,7 @@ describe('automatic host preparation', () => {
|
||||
const t5 = await publish('t5');
|
||||
const t6 = await publish('t6');
|
||||
const readyInventory = {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
|
||||
capabilities: [
|
||||
...inventory.capabilities!,
|
||||
@@ -195,7 +195,7 @@ describe('automatic host preparation', () => {
|
||||
it('requires both games when enabling even if one library is already cached', async () => {
|
||||
const version = await publish('t5');
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' }]
|
||||
});
|
||||
verified(id, version);
|
||||
@@ -212,7 +212,7 @@ describe('automatic host preparation', () => {
|
||||
const t5 = await publish('t5');
|
||||
const t6 = await publish('t6');
|
||||
const readyInventory = {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
|
||||
};
|
||||
const id = enroll(readyInventory);
|
||||
@@ -310,7 +310,11 @@ describe('automatic host preparation', () => {
|
||||
expect(operations(host.id)).toHaveLength(1);
|
||||
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
||||
const preparedInventory = {
|
||||
...inventory,
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraryCaches: []
|
||||
};
|
||||
const runtimeHeartbeat = await reportHeartbeat({
|
||||
request: request('heartbeat', { inventory: preparedInventory })
|
||||
});
|
||||
@@ -369,7 +373,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: ['another:runtime'] });
|
||||
const id = enroll({ preparedRuntimeImages: ['another:runtime'] });
|
||||
const host = hosts.getHost(id)!;
|
||||
const status = preparation.hostPreparationStatus(host);
|
||||
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
|
||||
@@ -406,7 +410,7 @@ describe('automatic host preparation', () => {
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
|
||||
heartbeat(id, { runtimeImages: [runtime] });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(2);
|
||||
const download = operations(id)[0];
|
||||
@@ -417,7 +421,7 @@ describe('automatic host preparation', () => {
|
||||
verified(id, version);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' }]
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||
@@ -432,7 +436,7 @@ describe('automatic host preparation', () => {
|
||||
game: 't5',
|
||||
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
||||
});
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const expected = new Map([latest, t6].map((version) => [version.id, version]));
|
||||
const downloaded: string[] = [];
|
||||
for (let index = 0; index < expected.size; index++) {
|
||||
@@ -452,7 +456,7 @@ describe('automatic host preparation', () => {
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(index + 1);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
|
||||
});
|
||||
}
|
||||
@@ -462,12 +466,15 @@ describe('automatic host preparation', () => {
|
||||
});
|
||||
|
||||
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toEqual([]);
|
||||
const old = await publish();
|
||||
verified(id, old);
|
||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||
const latest = await publish();
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -479,7 +486,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const first = operations(id)[0];
|
||||
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
||||
@@ -503,8 +510,8 @@ describe('automatic host preparation', () => {
|
||||
async (automatic) => {
|
||||
const old = await publish();
|
||||
const image = 'ghcr.io/example/plutainer:queued-version';
|
||||
const id = enroll({ runtimeImages: [runtime, image] });
|
||||
const other = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime, image] });
|
||||
const other = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const { queueNativeCreation } =
|
||||
await import('../../src/lib/server/provisioning/native.js');
|
||||
for (let index = 0; index < 2; index++) {
|
||||
@@ -521,7 +528,7 @@ describe('automatic host preparation', () => {
|
||||
).toBe(true);
|
||||
}
|
||||
// Also cover pre-existing intents whose image is no longer cached.
|
||||
heartbeat(id, { runtimeImages: [runtime] });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||
const latest = await publish();
|
||||
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
||||
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
|
||||
@@ -555,7 +562,7 @@ describe('automatic host preparation', () => {
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime, image],
|
||||
preparedRuntimeImages: [runtime, image],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -568,7 +575,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
@@ -604,7 +611,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const [first] = hosts.pendingHostOperations(id);
|
||||
const latest = await publish();
|
||||
@@ -613,7 +620,10 @@ describe('automatic host preparation', () => {
|
||||
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
||||
verified(id, old);
|
||||
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({
|
||||
state: 'pending',
|
||||
@@ -688,23 +698,26 @@ describe('automatic host preparation', () => {
|
||||
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||
});
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||
expect(operations(id)).toEqual([]);
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||
});
|
||||
|
||||
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const operation = libraries.queueLibraryDownload(id, version.id);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
|
||||
});
|
||||
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
||||
@@ -715,7 +728,7 @@ describe('automatic host preparation', () => {
|
||||
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
||||
const oversized = await publish();
|
||||
const small = await publish('t6', 'small');
|
||||
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 });
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
const status = preparation.reconcileHostPreparation(id);
|
||||
expect(status.items).toEqual(
|
||||
@@ -731,7 +744,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
libraries.updateHostLibraryCaches(id, [
|
||||
{
|
||||
libraryVersionId: version.id,
|
||||
@@ -741,7 +754,7 @@ describe('automatic host preparation', () => {
|
||||
}
|
||||
]);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||
@@ -753,7 +766,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const operation = operations(id)[0];
|
||||
libraries.updateHostLibraryCaches(id, [
|
||||
@@ -768,11 +781,11 @@ describe('automatic host preparation', () => {
|
||||
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
||||
const failedAt = Date.now();
|
||||
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
vi.setSystemTime(failedAt + 60_000);
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||
expect(operations(id)).toHaveLength(2);
|
||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||
@@ -800,7 +813,7 @@ describe('optional host Plutonium cache', () => {
|
||||
(state) => {
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -819,7 +832,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
||||
const id = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -842,7 +855,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const ready = {
|
||||
...inventory,
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||
};
|
||||
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
||||
@@ -903,7 +916,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }],
|
||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||
};
|
||||
@@ -948,7 +961,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const pull = operations(id)[0];
|
||||
expect(pull.kind).toBe('runtime.pull');
|
||||
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
||||
heartbeat(id, { ...capable, runtimeImages: [runtime] });
|
||||
heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
expect(download.kind).toBe('library.download');
|
||||
@@ -956,7 +969,7 @@ describe('optional host Plutonium cache', () => {
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||
};
|
||||
heartbeat(id, ready);
|
||||
@@ -1005,7 +1018,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5354, bytes: 25_000 }
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -1048,7 +1061,7 @@ describe('optional host Plutonium cache', () => {
|
||||
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
|
||||
});
|
||||
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
||||
const id = enroll({ ...capable, runtimeImages: [runtime] });
|
||||
const id = enroll({ ...capable, preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({
|
||||
kind: 'plutonium.prepare',
|
||||
@@ -1060,7 +1073,7 @@ describe('optional host Plutonium cache', () => {
|
||||
).toBe('Latest production revision');
|
||||
const cached = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5353, bytes: 200 }
|
||||
});
|
||||
preparation.reconcileHostPreparation(cached);
|
||||
@@ -1071,7 +1084,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('leaves agents without the cache capability unchanged', async () => {
|
||||
await release(5355);
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||
});
|
||||
expect(
|
||||
@@ -1084,7 +1097,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -1135,7 +1148,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const id = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||
});
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
|
||||
@@ -11,7 +11,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||
};
|
||||
beforeEach(async () => {
|
||||
|
||||
@@ -193,7 +193,7 @@ describe('native creation intents before placement', () => {
|
||||
memoryTotal: 32 * 1024 ** 3,
|
||||
memoryAvailable: 30 * 1024 ** 3,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
|
||||
@@ -12,7 +12,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||
};
|
||||
beforeEach(async () => {
|
||||
@@ -494,6 +494,21 @@ describe('native host enrollment and scoped authentication', () => {
|
||||
});
|
||||
|
||||
describe('native host readiness and reservations', () => {
|
||||
it.each([
|
||||
{ preparedRuntimeImages: 'plutainer:test' },
|
||||
{ preparedRuntimeImages: [123] },
|
||||
{ preparedRuntimeImages: null }
|
||||
])(
|
||||
'rejects malformed prepared runtime inventory: $preparedRuntimeImages',
|
||||
async ({ preparedRuntimeImages }) => {
|
||||
const api = await enrolled();
|
||||
expect(() =>
|
||||
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages })
|
||||
).toThrow(/runtime image inventory/i);
|
||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||
}
|
||||
);
|
||||
|
||||
it('separates a connected agent from ready Docker, storage, runtime and library state', async () => {
|
||||
const api = await enrolled();
|
||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||
@@ -518,8 +533,8 @@ describe('native host readiness and reservations', () => {
|
||||
});
|
||||
it.each([
|
||||
{ missing: { libraries: [] }, expected: 'needs-game-files' },
|
||||
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
|
||||
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||
{ missing: { preparedRuntimeImages: [] }, expected: 'needs-runtime' },
|
||||
{ missing: { preparedRuntimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||
])(
|
||||
'names missing content as $expected until a prepared heartbeat arrives',
|
||||
async ({ missing, expected }) => {
|
||||
@@ -545,7 +560,7 @@ describe('native host readiness and reservations', () => {
|
||||
download.id,
|
||||
runtime.id
|
||||
]);
|
||||
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
|
||||
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages: [], libraries: [] });
|
||||
expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([
|
||||
download.id,
|
||||
runtime.id
|
||||
|
||||
@@ -39,7 +39,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
enrollHost(createEnrollment(hostId).token, inventory);
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -43,7 +43,7 @@ const inventory = () => ({
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
|
||||
});
|
||||
beforeEach(async () => {
|
||||
|
||||
@@ -34,7 +34,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'published-game-version', state: 'ready' }],
|
||||
capabilities: ['instance.safety-override']
|
||||
});
|
||||
|
||||
@@ -47,7 +47,7 @@ async function heartbeat(hostId: string, cpuUsagePercent: number) {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * gib,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [{ id: libraryId, state: 'ready' }],
|
||||
instanceResources: instances.map((instance) => ({
|
||||
instanceId: instance.id,
|
||||
@@ -92,7 +92,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * gib,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [{ id: libraryId, state: 'ready' }]
|
||||
});
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
|
||||
@@ -548,7 +548,7 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [config.defaultRuntimeImage],
|
||||
preparedRuntimeImages: [config.defaultRuntimeImage],
|
||||
libraries: [{ id: 'lib', state: 'ready' }]
|
||||
});
|
||||
const generation = action === 'create' ? 1 : 2;
|
||||
|
||||
@@ -23,16 +23,31 @@ afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
});
|
||||
|
||||
async function cache(images: string[]) {
|
||||
async function reportImages(
|
||||
preparedImages: string[] | undefined,
|
||||
cachedImages = preparedImages ?? []
|
||||
) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.getHost(prepared.host.id)!;
|
||||
hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images });
|
||||
hosts.heartbeat(host.id, {
|
||||
...host.inventory,
|
||||
runtimeImages: cachedImages,
|
||||
preparedRuntimeImages: preparedImages
|
||||
});
|
||||
}
|
||||
|
||||
describe('runtime image launch policy', () => {
|
||||
it.each([false, true])(
|
||||
'rejects an uncached custom image at POST /api/servers (automatic: %s)',
|
||||
async (automatic) => {
|
||||
it.each([
|
||||
{ automatic: false, cached: false, legacy: false },
|
||||
{ automatic: true, cached: false, legacy: false },
|
||||
{ automatic: false, cached: true, legacy: false },
|
||||
{ automatic: true, cached: true, legacy: false },
|
||||
{ automatic: false, cached: true, legacy: true },
|
||||
{ automatic: true, cached: true, legacy: true }
|
||||
])(
|
||||
'rejects an unprepared custom image at POST /api/servers ($automatic, cached: $cached, legacy: $legacy)',
|
||||
async ({ automatic, cached, legacy }) => {
|
||||
await reportImages(legacy ? undefined : [], cached ? [custom] : []);
|
||||
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
||||
const response = await POST({
|
||||
request: new Request('https://controller.test/api/servers', {
|
||||
@@ -57,13 +72,88 @@ describe('runtime image launch policy', () => {
|
||||
);
|
||||
|
||||
it('allows the configured default before it is cached', async () => {
|
||||
await cache([]);
|
||||
await reportImages([]);
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
expect(queueNativeCreation(launch).ok).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the cached configured default eligible with a legacy agent', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
||||
expect(nativeSelection({}, 't5').hostId).toBe(prepared.host.id);
|
||||
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
expect(getHost(prepared.host.id)!.inventory.preparedRuntimeImages).toEqual([
|
||||
config.defaultRuntimeImage
|
||||
]);
|
||||
});
|
||||
|
||||
it('ignores malformed preparation fields already stored by an older controller', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const inventory = getHost(prepared.host.id)!.inventory;
|
||||
for (const reported of [{}, 42, custom]) {
|
||||
db()
|
||||
.prepare('UPDATE ec_hosts SET inventory=? WHERE id=?')
|
||||
.run(
|
||||
JSON.stringify({
|
||||
...inventory,
|
||||
runtimeImages: [config.defaultRuntimeImage, custom],
|
||||
preparedRuntimeImages: reported
|
||||
}),
|
||||
prepared.host.id
|
||||
);
|
||||
expect(getHost(prepared.host.id)).toMatchObject({
|
||||
readiness: 'ready',
|
||||
inventory: { preparedRuntimeImages: [config.defaultRuntimeImage] }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('hides unrelated cached images from host list, detail, and page responses', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||
const list = await import('../../src/routes/api/hosts/+server.js');
|
||||
const detail = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||
const { load } = await import('../../src/routes/hosts/+page.server.js');
|
||||
const request = new Request('https://controller.test/api/hosts');
|
||||
const bodies = [
|
||||
await list.GET({ request }).json(),
|
||||
await detail.GET({ request, params: { id: prepared.host.id } }).json(),
|
||||
load()
|
||||
];
|
||||
for (const body of bodies) {
|
||||
const host = 'hosts' in body ? body.hosts[0] : body.host;
|
||||
expect(host.inventory.runtimeImages).toBeUndefined();
|
||||
expect(host.inventory.preparedRuntimeImages).toEqual([config.defaultRuntimeImage]);
|
||||
expect(JSON.stringify(body)).not.toContain(custom);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not place an accepted runtime on another host that only caches it', async () => {
|
||||
await reportImages([custom]);
|
||||
const other = await prepareNativeHost();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
hosts.heartbeat(other.host.id, {
|
||||
...hosts.getHost(other.host.id)!.inventory,
|
||||
runtimeImages: [custom]
|
||||
});
|
||||
expect(() => hosts.requireReadyHost(other.host.id, { image: custom })).toThrow(
|
||||
/runtime image is not prepared/
|
||||
);
|
||||
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
||||
expect(() =>
|
||||
choosePlacement({
|
||||
hostId: other.host.id,
|
||||
libraryId: other.versions.get('t5')!,
|
||||
image: custom
|
||||
})
|
||||
).toThrow(/runtime image is not prepared/);
|
||||
});
|
||||
|
||||
it('accepts prepared images only on hosts that have them', async () => {
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const other = hosts.createHost({ name: 'Unprepared host' });
|
||||
const { nativeLaunchSelection, nativeSelection } =
|
||||
@@ -81,7 +171,7 @@ describe('runtime image launch policy', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('requires exact image references and does not treat a queued admin pull as cached', async () => {
|
||||
it('requires exact prepared references and does not treat a queued admin pull as preparation', async () => {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom });
|
||||
const { nativeLaunchSelection } =
|
||||
@@ -89,25 +179,25 @@ describe('runtime image launch policy', () => {
|
||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
await cache([custom + '-other']);
|
||||
await reportImages([custom + '-other']);
|
||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
});
|
||||
|
||||
it('rechecks cached images when submitting a saved single-server preview', async () => {
|
||||
await cache([custom]);
|
||||
it('rechecks preparation when submitting a saved preview even if the image is still cached', async () => {
|
||||
await reportImages([custom]);
|
||||
const { previewNativeCreation, queueNativeCreation } =
|
||||
await import('../../src/lib/server/provisioning/native.js');
|
||||
const preview = previewNativeCreation({ ...launch, image: custom });
|
||||
await cache([]);
|
||||
await reportImages([], [custom]);
|
||||
expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({
|
||||
ok: false,
|
||||
message: expect.stringContaining('prepared by an admin')
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => {
|
||||
it('rejects unprepared collection images and rechecks saved previews before writing any jobs', async () => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } =
|
||||
await import('../../src/lib/server/fleet/collections.js');
|
||||
@@ -115,15 +205,16 @@ describe('runtime image launch policy', () => {
|
||||
name: 'Image policy',
|
||||
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
||||
});
|
||||
await reportImages([], [custom]);
|
||||
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
|
||||
const keys = Object.fromEntries(
|
||||
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
||||
);
|
||||
await cache([]);
|
||||
await reportImages([], [custom]);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
@@ -136,7 +227,7 @@ describe('runtime image launch policy', () => {
|
||||
]) {
|
||||
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
}
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user