This commit is contained in:
+10
-4
@@ -35,7 +35,7 @@ import urllib.parse
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
VERSION = "1.8.5"
|
VERSION = "1.8.6"
|
||||||
LABEL = "io.electric-cherry"
|
LABEL = "io.electric-cherry"
|
||||||
MAX_FILE = 32 * 1024 * 1024
|
MAX_FILE = 32 * 1024 * 1024
|
||||||
MANIFEST_MAX_ENTRIES = 10000
|
MANIFEST_MAX_ENTRIES = 10000
|
||||||
@@ -1462,7 +1462,7 @@ class Agent:
|
|||||||
progress = list(progress.values())
|
progress = list(progress.values())
|
||||||
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
||||||
caches.extend(progress)
|
caches.extend(progress)
|
||||||
images = []
|
images, prepared_images = [], []
|
||||||
if docker_running:
|
if docker_running:
|
||||||
with contextlib.suppress(AgentError):
|
with contextlib.suppress(AgentError):
|
||||||
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
||||||
@@ -1480,7 +1480,13 @@ class Agent:
|
|||||||
aliases = self.root / "agent" / "runtime-images.json"
|
aliases = self.root / "agent" / "runtime-images.json"
|
||||||
if aliases.exists():
|
if aliases.exists():
|
||||||
with contextlib.suppress(OSError, ValueError):
|
with contextlib.suppress(OSError, ValueError):
|
||||||
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
|
saved = json.loads(aliases.read_text())
|
||||||
|
if isinstance(saved, dict):
|
||||||
|
# Only runtime.pull records administrator-prepared
|
||||||
|
# references. Other Docker cache entries are useful
|
||||||
|
# diagnostics, but do not grant launch permission.
|
||||||
|
prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids]
|
||||||
|
images.extend(prepared_images)
|
||||||
resources = {}
|
resources = {}
|
||||||
if docker_running:
|
if docker_running:
|
||||||
observations = []
|
observations = []
|
||||||
@@ -1499,7 +1505,7 @@ class Agent:
|
|||||||
update = self.update_inventory()
|
update = self.update_inventory()
|
||||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||||
plutonium_cache = self.plutonium_cache_inventory()
|
plutonium_cache = self.plutonium_cache_inventory()
|
||||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||||
|
|
||||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||||
health = self.inspect(instance_id, generation)
|
health = self.inspect(instance_id, generation)
|
||||||
|
|||||||
@@ -2502,12 +2502,62 @@ class AgentTests(unittest.TestCase):
|
|||||||
return f"ghcr.io/ayymoss/plutainer|latest|{image_digest}|{image_id}\n"
|
return f"ghcr.io/ayymoss/plutainer|latest|{image_digest}|{image_id}\n"
|
||||||
return original(*args, **kwargs)
|
return original(*args, **kwargs)
|
||||||
self.docker.call = call
|
self.docker.call = call
|
||||||
module.atomic_json(self.agent.root / "agent/runtime-images.json", {"ghcr.io/ayymoss/plutainer": image_id, "removed-image": "missing-id"})
|
prepared = ["ghcr.io/ayymoss/plutainer", "ghcr.io/ayymoss/plutainer@" + image_digest]
|
||||||
images = self.agent.inventory()["runtimeImages"]
|
module.atomic_json(self.agent.root / "agent/runtime-images.json", {**dict.fromkeys(prepared, image_id), "removed-image": "missing-id"})
|
||||||
|
inventory = self.agent.inventory()
|
||||||
|
images = inventory["runtimeImages"]
|
||||||
self.assertIn("ghcr.io/ayymoss/plutainer@" + image_digest, images)
|
self.assertIn("ghcr.io/ayymoss/plutainer@" + image_digest, images)
|
||||||
self.assertIn("ghcr.io/ayymoss/plutainer:latest", images)
|
self.assertIn("ghcr.io/ayymoss/plutainer:latest", images)
|
||||||
self.assertIn("ghcr.io/ayymoss/plutainer", images)
|
self.assertIn("ghcr.io/ayymoss/plutainer", images)
|
||||||
self.assertNotIn("removed-image", images)
|
self.assertNotIn("removed-image", images)
|
||||||
|
self.assertEqual(inventory["preparedRuntimeImages"], prepared)
|
||||||
|
|
||||||
|
def test_runtime_pull_prepares_only_the_requested_reference_from_the_docker_cache(self):
|
||||||
|
image = "registry.example/custom-runtime:stable"
|
||||||
|
image_id = "sha256:" + "1" * 64
|
||||||
|
original = self.docker.call
|
||||||
|
|
||||||
|
def call(*args, **kwargs):
|
||||||
|
if args[:2] == ("image", "ls"):
|
||||||
|
return (f"registry.example/custom-runtime|stable|<none>|{image_id}\n"
|
||||||
|
f"registry.example/custom-runtime|unprepared-alias|<none>|{image_id}\n"
|
||||||
|
"unrelated-service|latest|<none>|sha256:unrelated\n")
|
||||||
|
if args[:2] == ("image", "inspect"):
|
||||||
|
self.assertEqual(args[-1], image)
|
||||||
|
return image_id
|
||||||
|
return original(*args, **kwargs)
|
||||||
|
|
||||||
|
self.docker.call = call
|
||||||
|
before = self.agent.inventory()
|
||||||
|
self.assertIn(image, before["runtimeImages"])
|
||||||
|
self.assertIn("unrelated-service:latest", before["runtimeImages"])
|
||||||
|
self.assertEqual(before["preparedRuntimeImages"], [])
|
||||||
|
|
||||||
|
result = self.agent.execute(self.operation("runtime.pull", {"image": image}))
|
||||||
|
self.assertEqual(result["state"], "completed")
|
||||||
|
inventory = self.agent.inventory()
|
||||||
|
self.assertEqual(inventory["runtimeImages"], before["runtimeImages"])
|
||||||
|
self.assertEqual(inventory["preparedRuntimeImages"], [image])
|
||||||
|
|
||||||
|
def test_runtime_preparation_inventory_fails_closed_when_records_or_docker_are_unavailable(self):
|
||||||
|
aliases = self.agent.root / "agent/runtime-images.json"
|
||||||
|
original = self.docker.call
|
||||||
|
|
||||||
|
def call(*args, **kwargs):
|
||||||
|
if args[:2] == ("image", "ls"):
|
||||||
|
return "custom-runtime|latest|<none>|sha256:prepared\n"
|
||||||
|
return original(*args, **kwargs)
|
||||||
|
|
||||||
|
self.docker.call = call
|
||||||
|
for content in ("invalid json", "[]", '{"custom-runtime:latest": []}', '{"custom-runtime:latest": "sha256:removed"}'):
|
||||||
|
with self.subTest(content=content):
|
||||||
|
aliases.write_text(content)
|
||||||
|
inventory = self.agent.inventory()
|
||||||
|
self.assertEqual(inventory["preparedRuntimeImages"], [])
|
||||||
|
self.assertEqual(inventory["runtimeImages"], ["custom-runtime:latest"])
|
||||||
|
module.atomic_json(aliases, {"custom-runtime:latest": "sha256:prepared"})
|
||||||
|
with patch.object(self.docker, "call", side_effect=AgentError("Docker unavailable")):
|
||||||
|
self.assertEqual(self.agent.inventory()["preparedRuntimeImages"], [])
|
||||||
|
|
||||||
def test_heartbeat_applies_configured_host_port_range_before_deployment(self):
|
def test_heartbeat_applies_configured_host_port_range_before_deployment(self):
|
||||||
self.agent.request = lambda *_: {"host": {"portMin": 30000, "portMax": 30100}}
|
self.agent.request = lambda *_: {"host": {"portMin": 30000, "portMax": 30100}}
|
||||||
|
|||||||
@@ -243,10 +243,15 @@ servers. Existing API clients may still explicitly constrain a launch to a host.
|
|||||||
Queued jobs retain their accepted game-file version. Automatic host preparation
|
Queued jobs retain their accepted game-file version. Automatic host preparation
|
||||||
also downloads the game files needed by unplaced jobs; a newer upload does not
|
also downloads the game files needed by unplaced jobs; a newer upload does not
|
||||||
remove files still needed by the queue. Only the configured default runtime is
|
remove files still needed by the queue. Only the configured default runtime is
|
||||||
pulled automatically. Custom images must already be cached on the selected host,
|
pulled automatically. Custom images must have been prepared by an admin from the
|
||||||
or on an active host for automatic placement, before a launch is accepted. Admins
|
selected host's page, or on an active host for automatic placement, before a launch
|
||||||
can prepare them from the host page. Queued launches and migrations never trigger
|
is accepted. An image cached by another Docker workload does not qualify. Agent
|
||||||
custom image pulls, and container creation cannot implicitly pull a missing image.
|
1.8.6 reports these exact prepared references from `runtime-images.json` while
|
||||||
|
their image IDs remain cached; the controller also trusts its configured default
|
||||||
|
when cached. Host pages and APIs expose only prepared runtimes. Older agents can
|
||||||
|
still run the configured default, but must update before accepting custom runtimes.
|
||||||
|
Queued launches and migrations never trigger custom image pulls, and container
|
||||||
|
creation cannot implicitly pull a missing image.
|
||||||
|
|
||||||
### Resource samples and learned allowances
|
### Resource samples and learned allowances
|
||||||
|
|
||||||
|
|||||||
@@ -163,7 +163,7 @@ export function seedFixtureData(handle: Database.Database, now: number): void {
|
|||||||
storageAvailable: 240 * GIB,
|
storageAvailable: 240 * GIB,
|
||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
runtimeImages: [image],
|
preparedRuntimeImages: [image],
|
||||||
libraries: [
|
libraries: [
|
||||||
{ id: 'fixture-t5', state: 'ready' },
|
{ id: 'fixture-t5', state: 'ready' },
|
||||||
{ id: 'fixture-t6', state: 'ready' }
|
{ id: 'fixture-t6', state: 'ready' }
|
||||||
|
|||||||
@@ -156,7 +156,10 @@ export interface HostInventory {
|
|||||||
dockerVersion?: string;
|
dockerVersion?: string;
|
||||||
dockerRunning?: boolean;
|
dockerRunning?: boolean;
|
||||||
directoriesWritable?: boolean;
|
directoriesWritable?: boolean;
|
||||||
|
/** Agent-reported Docker cache; removed from presented host inventory. */
|
||||||
runtimeImages?: string[];
|
runtimeImages?: string[];
|
||||||
|
/** Prepared, locally available image references eligible for launches. */
|
||||||
|
preparedRuntimeImages?: string[];
|
||||||
libraries?: HostLibrary[];
|
libraries?: HostLibrary[];
|
||||||
currentOperation?: string | null;
|
currentOperation?: string | null;
|
||||||
[key: string]: unknown;
|
[key: string]: unknown;
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ export function launchAvailability(selections: NativePlacement[]): {
|
|||||||
);
|
);
|
||||||
const eligible = (candidate: HostAvailability, selection: NativePlacement) =>
|
const eligible = (candidate: HostAvailability, selection: NativePlacement) =>
|
||||||
(!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) &&
|
(!selection.hostId || selection.automatic || candidate.host.id === selection.hostId) &&
|
||||||
!!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '') &&
|
!!candidate.host.inventory.preparedRuntimeImages?.includes(selection.image ?? '') &&
|
||||||
!!candidate.host.inventory.libraries?.some(
|
!!candidate.host.inventory.libraries?.some(
|
||||||
(library) =>
|
(library) =>
|
||||||
library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)
|
library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state)
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
|
|||||||
if (!host.credentialActive && host.enrolled)
|
if (!host.credentialActive && host.enrolled)
|
||||||
reasons.push('Host credentials are inactive.');
|
reasons.push('Host credentials are inactive.');
|
||||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||||
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage))
|
if (!host.inventory.preparedRuntimeImages?.includes(config.defaultRuntimeImage))
|
||||||
reasons.push('The default runtime is not ready.');
|
reasons.push('The default runtime is not ready.');
|
||||||
if (!requestedGames.some((game) => games.includes(game)))
|
if (!requestedGames.some((game) => games.includes(game)))
|
||||||
reasons.push('Latest game files for this selection are not ready.');
|
reasons.push('Latest game files for this selection are not ready.');
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ function hostFromRow(row: HostRow): Host {
|
|||||||
)
|
)
|
||||||
reasons.push('Insufficient available game storage.');
|
reasons.push('Insufficient available game storage.');
|
||||||
const infrastructureReady = reasons.length === 0;
|
const infrastructureReady = reasons.length === 0;
|
||||||
const runtimeReady = !!inventory.runtimeImages?.length;
|
const runtimeReady = !!inventory.preparedRuntimeImages?.length;
|
||||||
const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready');
|
const libraryReady = !!inventory.libraries?.some((library) => library.state === 'ready');
|
||||||
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
|
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
|
||||||
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
|
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
|
||||||
@@ -676,12 +676,14 @@ export function validateInventory(value: unknown): HostInventory {
|
|||||||
for (const key of ['dockerRunning', 'directoriesWritable'])
|
for (const key of ['dockerRunning', 'directoriesWritable'])
|
||||||
if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean')
|
if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean')
|
||||||
throw new Error(`Invalid inventory ${key}.`);
|
throw new Error(`Invalid inventory ${key}.`);
|
||||||
|
for (const key of ['runtimeImages', 'preparedRuntimeImages'] as const) {
|
||||||
|
const images = inventory[key];
|
||||||
if (
|
if (
|
||||||
inventory.runtimeImages !== undefined &&
|
images !== undefined &&
|
||||||
(!Array.isArray(inventory.runtimeImages) ||
|
(!Array.isArray(images) || images.some((image) => typeof image !== 'string'))
|
||||||
inventory.runtimeImages.some((image) => typeof image !== 'string'))
|
|
||||||
)
|
)
|
||||||
throw new Error('Invalid runtime image inventory.');
|
throw new Error('Invalid runtime image inventory.');
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
inventory.capabilities !== undefined &&
|
inventory.capabilities !== undefined &&
|
||||||
(!Array.isArray(inventory.capabilities) ||
|
(!Array.isArray(inventory.capabilities) ||
|
||||||
@@ -952,8 +954,8 @@ export function requireReadyHost(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
reasons.push('The selected game library is not ready on this host.');
|
reasons.push('The selected game library is not ready on this host.');
|
||||||
if (options.image && !host.inventory.runtimeImages?.includes(options.image))
|
if (options.image && !host.inventory.preparedRuntimeImages?.includes(options.image))
|
||||||
reasons.push('The selected runtime image is not cached on this host.');
|
reasons.push('The selected runtime image is not prepared on this host.');
|
||||||
if (
|
if (
|
||||||
options.port !== undefined &&
|
options.port !== undefined &&
|
||||||
!hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)
|
!hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)
|
||||||
|
|||||||
@@ -190,7 +190,7 @@ function targets(host: Host): Target[] {
|
|||||||
const result: Target[] = [config.defaultRuntimeImage].map((image) => {
|
const result: Target[] = [config.defaultRuntimeImage].map((image) => {
|
||||||
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
||||||
const runtimeReady =
|
const runtimeReady =
|
||||||
host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
host.inventory.preparedRuntimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||||
return {
|
return {
|
||||||
image,
|
image,
|
||||||
requiredBytes: host.minimumFreeBytes,
|
requiredBytes: host.minimumFreeBytes,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||||
|
import { config } from '../config.js';
|
||||||
import type { HostInventory } from '../../hosts.js';
|
import type { HostInventory } from '../../hosts.js';
|
||||||
|
|
||||||
type InventoryCache = Map<string, { json: string; inventory: HostInventory }>;
|
type InventoryCache = Map<string, { json: string; inventory: HostInventory }>;
|
||||||
@@ -21,7 +22,20 @@ export function parseHostInventory(hostId: string, json: string): HostInventory
|
|||||||
const cache = requests.getStore()?.inventories;
|
const cache = requests.getStore()?.inventories;
|
||||||
const saved = cache?.get(hostId);
|
const saved = cache?.get(hostId);
|
||||||
if (saved?.json === json) return saved.inventory;
|
if (saved?.json === json) return saved.inventory;
|
||||||
const inventory: HostInventory = JSON.parse(json);
|
// Full Docker inventory can include unrelated workloads. Keep it out of
|
||||||
|
// every presented host, including inventory persisted by older agents.
|
||||||
|
const { runtimeImages, ...inventory }: HostInventory = JSON.parse(json);
|
||||||
|
// The configured default is explicitly trusted by the controller. Older
|
||||||
|
// agents may prove it is cached, but cannot authorize any custom runtime.
|
||||||
|
const reported = inventory.preparedRuntimeImages;
|
||||||
|
const prepared = new Set(
|
||||||
|
Array.isArray(reported) && reported.every((image) => typeof image === 'string')
|
||||||
|
? reported
|
||||||
|
: []
|
||||||
|
);
|
||||||
|
if (runtimeImages?.includes(config.defaultRuntimeImage))
|
||||||
|
prepared.add(config.defaultRuntimeImage);
|
||||||
|
inventory.preparedRuntimeImages = [...prepared];
|
||||||
// Compare the stored JSON so heartbeats and transaction rollbacks within
|
// Compare the stored JSON so heartbeats and transaction rollbacks within
|
||||||
// a request remain visible. Other host state is always read afresh.
|
// a request remain visible. Other host state is always read afresh.
|
||||||
cache?.set(hostId, { json, inventory });
|
cache?.set(hostId, { json, inventory });
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ export function nativeLaunchSelection(
|
|||||||
throw new Error('The selected host is being removed. Choose another host.');
|
throw new Error('The selected host is being removed. Choose another host.');
|
||||||
}
|
}
|
||||||
// Only the controller-configured default may be prepared automatically.
|
// Only the controller-configured default may be prepared automatically.
|
||||||
// Custom runtimes must already be present in trusted host inventory; a
|
// Custom runtimes must be reported as admin-prepared by the host agent; a
|
||||||
// launch request must never authorize a registry download.
|
// launch request must never authorize a registry download.
|
||||||
if (
|
if (
|
||||||
image !== config.defaultRuntimeImage &&
|
image !== config.defaultRuntimeImage &&
|
||||||
@@ -144,7 +144,7 @@ export function nativeLaunchSelection(
|
|||||||
(host) =>
|
(host) =>
|
||||||
!host.decommission &&
|
!host.decommission &&
|
||||||
(automatic ? host.mode === 'active' : host.id === input.hostId) &&
|
(automatic ? host.mode === 'active' : host.id === input.hostId) &&
|
||||||
host.inventory.runtimeImages?.includes(image)
|
host.inventory.preparedRuntimeImages?.includes(image)
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
|||||||
@@ -45,7 +45,7 @@
|
|||||||
hosts.filter(
|
hosts.filter(
|
||||||
(host) =>
|
(host) =>
|
||||||
host.readiness === 'ready' &&
|
host.readiness === 'ready' &&
|
||||||
host.inventory.runtimeImages?.includes(selection.image)
|
host.inventory.preparedRuntimeImages?.includes(selection.image)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
const latest = $derived(latestGameFiles(libraries));
|
const latest = $derived(latestGameFiles(libraries));
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ export async function GET() {
|
|||||||
cache.status === 'ready' && readyHosts.some((host) => host.id === cache.hostId)
|
cache.status === 'ready' && readyHosts.some((host) => host.id === cache.hostId)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
const runtimes = connected.filter((host) => host.inventory.runtimeImages?.length);
|
const runtimes = connected.filter((host) => host.inventory.preparedRuntimeImages?.length);
|
||||||
const checks: SetupCheck[] = [
|
const checks: SetupCheck[] = [
|
||||||
{
|
{
|
||||||
id: 'hosts',
|
id: 'hosts',
|
||||||
@@ -46,7 +46,7 @@ export async function GET() {
|
|||||||
title: 'Plutainer runtime',
|
title: 'Plutainer runtime',
|
||||||
state: runtimes.length ? 'ready' : 'attention',
|
state: runtimes.length ? 'ready' : 'attention',
|
||||||
detail: runtimes.length
|
detail: runtimes.length
|
||||||
? `${runtimes.length} connected hosts report cached runtime images.`
|
? `${runtimes.length} connected hosts report prepared runtime images.`
|
||||||
: 'Hosts prepare the configured Plutainer runtime automatically.',
|
: 'Hosts prepare the configured Plutainer runtime automatically.',
|
||||||
href: '/hosts',
|
href: '/hosts',
|
||||||
action: 'View host setup'
|
action: 'View host setup'
|
||||||
|
|||||||
@@ -447,7 +447,7 @@
|
|||||||
</li>{/each}
|
</li>{/each}
|
||||||
</ul>{:else}<p>No cached game libraries reported yet.</p>{/if}
|
</ul>{:else}<p>No cached game libraries reported yet.</p>{/if}
|
||||||
<p>
|
<p>
|
||||||
Cached images: {host.inventory.runtimeImages?.join(', ') ||
|
Prepared runtimes: {host.inventory.preparedRuntimeImages?.join(', ') ||
|
||||||
'None reported yet'}
|
'None reported yet'}
|
||||||
</p>
|
</p>
|
||||||
{#if canManage}<details>
|
{#if canManage}<details>
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ import urllib.parse
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
VERSION = "1.8.5"
|
VERSION = "1.8.6"
|
||||||
LABEL = "io.electric-cherry"
|
LABEL = "io.electric-cherry"
|
||||||
MAX_FILE = 32 * 1024 * 1024
|
MAX_FILE = 32 * 1024 * 1024
|
||||||
MANIFEST_MAX_ENTRIES = 10000
|
MANIFEST_MAX_ENTRIES = 10000
|
||||||
@@ -1462,7 +1462,7 @@ class Agent:
|
|||||||
progress = list(progress.values())
|
progress = list(progress.values())
|
||||||
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
libraries.extend({"id": item["libraryVersionId"], "state": item["status"]} for item in progress)
|
||||||
caches.extend(progress)
|
caches.extend(progress)
|
||||||
images = []
|
images, prepared_images = [], []
|
||||||
if docker_running:
|
if docker_running:
|
||||||
with contextlib.suppress(AgentError):
|
with contextlib.suppress(AgentError):
|
||||||
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
lines = self.docker.call("image", "ls", "--no-trunc", "--digests", "--format", "{{.Repository}}|{{.Tag}}|{{.Digest}}|{{.ID}}", timeout=15).splitlines()
|
||||||
@@ -1480,7 +1480,13 @@ class Agent:
|
|||||||
aliases = self.root / "agent" / "runtime-images.json"
|
aliases = self.root / "agent" / "runtime-images.json"
|
||||||
if aliases.exists():
|
if aliases.exists():
|
||||||
with contextlib.suppress(OSError, ValueError):
|
with contextlib.suppress(OSError, ValueError):
|
||||||
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
|
saved = json.loads(aliases.read_text())
|
||||||
|
if isinstance(saved, dict):
|
||||||
|
# Only runtime.pull records administrator-prepared
|
||||||
|
# references. Other Docker cache entries are useful
|
||||||
|
# diagnostics, but do not grant launch permission.
|
||||||
|
prepared_images = [alias for alias, image_id in saved.items() if isinstance(image_id, str) and image_id in image_ids]
|
||||||
|
images.extend(prepared_images)
|
||||||
resources = {}
|
resources = {}
|
||||||
if docker_running:
|
if docker_running:
|
||||||
observations = []
|
observations = []
|
||||||
@@ -1499,7 +1505,7 @@ class Agent:
|
|||||||
update = self.update_inventory()
|
update = self.update_inventory()
|
||||||
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||||
plutonium_cache = self.plutonium_cache_inventory()
|
plutonium_cache = self.plutonium_cache_inventory()
|
||||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||||
|
|
||||||
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
def safe_to_disrupt(self, instance_id, generation, *, allow_unhealthy=False):
|
||||||
health = self.inspect(instance_id, generation)
|
health = self.inspect(instance_id, generation)
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"version": "1.8.5",
|
"version": "1.8.6",
|
||||||
"protocolVersion": 1,
|
"protocolVersion": 1,
|
||||||
"platform": "debian-amd64",
|
"platform": "debian-amd64",
|
||||||
"sha256": "56187a244b551af884db6bd6419daf7dd7d931fa1e5d5c5414559bb6e25f1094",
|
"sha256": "79702dc8cf80d2521153bfda9241f7240285014fe746fc921f0a7793dd81212b",
|
||||||
"updaterProtocolVersion": 1,
|
"updaterProtocolVersion": 1,
|
||||||
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
|
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ export async function prepareNativeHost(servers: string[] = []) {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100_000_000_000,
|
storageAvailable: 100_000_000_000,
|
||||||
runtimeImages: [config.defaultRuntimeImage]
|
preparedRuntimeImages: [config.defaultRuntimeImage]
|
||||||
};
|
};
|
||||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
||||||
const versions = new Map<string, string>();
|
const versions = new Map<string, string>();
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ const inventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: ['plutainer:test'],
|
preparedRuntimeImages: ['plutainer:test'],
|
||||||
libraries: [{ id: 'test-library', state: 'ready' }],
|
libraries: [{ id: 'test-library', state: 'ready' }],
|
||||||
capabilities: ['instance.cancel-create', 'operations.health-lane']
|
capabilities: ['instance.cancel-create', 'operations.health-lane']
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ const inventory = {
|
|||||||
cpuUsagePercent: 0,
|
cpuUsagePercent: 0,
|
||||||
memoryTotal: 64 * GiB,
|
memoryTotal: 64 * GiB,
|
||||||
memoryAvailable: 60 * GiB,
|
memoryAvailable: 60 * GiB,
|
||||||
runtimeImages: [image],
|
preparedRuntimeImages: [image],
|
||||||
libraries: [
|
libraries: [
|
||||||
{ id: 't5-version', state: 'ready' },
|
{ id: 't5-version', state: 'ready' },
|
||||||
{ id: 't6-version', state: 'ready' }
|
{ id: 't6-version', state: 'ready' }
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 1_000_000_000_000,
|
storageAvailable: 1_000_000_000_000,
|
||||||
runtimeImages: [image]
|
preparedRuntimeImages: [image]
|
||||||
};
|
};
|
||||||
enrollHost(createEnrollment(hostId).token, inventory);
|
enrollHost(createEnrollment(hostId).token, inventory);
|
||||||
const { createLibraryImport, updateHostLibraryCaches } =
|
const { createLibraryImport, updateHostLibraryCaches } =
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 1_000_000_000_000,
|
storageAvailable: 1_000_000_000_000,
|
||||||
runtimeImages: [image]
|
preparedRuntimeImages: [image]
|
||||||
};
|
};
|
||||||
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
||||||
const { createLibraryImport, updateHostLibraryCaches } =
|
const { createLibraryImport, updateHostLibraryCaches } =
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: [runtime]
|
preparedRuntimeImages: [runtime]
|
||||||
});
|
});
|
||||||
version = await publish();
|
version = await publish();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ const inventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: ['image'],
|
preparedRuntimeImages: ['image'],
|
||||||
libraries: [{ id: 'library', state: 'ready' }]
|
libraries: [{ id: 'library', state: 'ready' }]
|
||||||
};
|
};
|
||||||
const lane: HostAgentLane = {
|
const lane: HostAgentLane = {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ const inventory = {
|
|||||||
cpuUsagePercent: 0,
|
cpuUsagePercent: 0,
|
||||||
memoryTotal: 64 * GiB,
|
memoryTotal: 64 * GiB,
|
||||||
memoryAvailable: 60 * GiB,
|
memoryAvailable: 60 * GiB,
|
||||||
runtimeImages: ['image'],
|
preparedRuntimeImages: ['image'],
|
||||||
libraries: [
|
libraries: [
|
||||||
{ id: 't5-library', state: 'ready' },
|
{ id: 't5-library', state: 'ready' },
|
||||||
{ id: 't6-library', state: 'ready' }
|
{ id: 't6-library', state: 'ready' }
|
||||||
@@ -46,6 +46,32 @@ afterEach(async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('cached launch availability', () => {
|
describe('cached launch availability', () => {
|
||||||
|
it.each([{ preparedRuntimeImages: [] }, { preparedRuntimeImages: ['another:image'] }])(
|
||||||
|
'excludes a cached image until it is prepared ($preparedRuntimeImages)',
|
||||||
|
async ({ preparedRuntimeImages }) => {
|
||||||
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||||
|
const { launchAvailability } =
|
||||||
|
await import('../../src/lib/server/hosts/availability.js');
|
||||||
|
heartbeat(hostId, {
|
||||||
|
...inventory,
|
||||||
|
runtimeImages: ['image'],
|
||||||
|
preparedRuntimeImages
|
||||||
|
});
|
||||||
|
const unavailable = launchAvailability([t5]);
|
||||||
|
expect(unavailable.availability).toMatchObject({
|
||||||
|
estimatedAvailable: 0,
|
||||||
|
portsAvailable: 0
|
||||||
|
});
|
||||||
|
expect(unavailable.warnings.join(' ')).toContain('Waiting for an eligible host');
|
||||||
|
|
||||||
|
heartbeat(hostId, { ...inventory, runtimeImages: ['image'] });
|
||||||
|
expect(launchAvailability([t5]).availability).toMatchObject({
|
||||||
|
estimatedAvailable: 1,
|
||||||
|
portsAvailable: 3
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
|
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
|
||||||
const { db } = await import('../../src/lib/server/db/index.js');
|
const { db } = await import('../../src/lib/server/db/index.js');
|
||||||
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ const inventory = {
|
|||||||
protocolVersion: 1,
|
protocolVersion: 1,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * GiB,
|
storageAvailable: 100 * GiB,
|
||||||
runtimeImages: ['image'],
|
preparedRuntimeImages: ['image'],
|
||||||
libraries: [{ id: 'library', state: 'ready' }]
|
libraries: [{ id: 'library', state: 'ready' }]
|
||||||
};
|
};
|
||||||
const sample = {
|
const sample = {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ const inventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: ['plutainer:test'],
|
preparedRuntimeImages: ['plutainer:test'],
|
||||||
libraries: [{ id: 'library', state: 'ready' }]
|
libraries: [{ id: 'library', state: 'ready' }]
|
||||||
};
|
};
|
||||||
let fixture: Fixture;
|
let fixture: Fixture;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ const inventory: HostInventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: [],
|
preparedRuntimeImages: [],
|
||||||
libraries: [],
|
libraries: [],
|
||||||
capabilities: ['host.decommission']
|
capabilities: ['host.decommission']
|
||||||
};
|
};
|
||||||
@@ -123,7 +123,7 @@ describe('automatic host preparation', () => {
|
|||||||
const t5 = await publish('t5');
|
const t5 = await publish('t5');
|
||||||
const t6 = await publish('t6');
|
const t6 = await publish('t6');
|
||||||
const readyInventory = {
|
const readyInventory = {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
|
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
|
||||||
capabilities: [
|
capabilities: [
|
||||||
...inventory.capabilities!,
|
...inventory.capabilities!,
|
||||||
@@ -195,7 +195,7 @@ describe('automatic host preparation', () => {
|
|||||||
it('requires both games when enabling even if one library is already cached', async () => {
|
it('requires both games when enabling even if one library is already cached', async () => {
|
||||||
const version = await publish('t5');
|
const version = await publish('t5');
|
||||||
const id = enroll({
|
const id = enroll({
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: version.id, state: 'ready' }]
|
libraries: [{ id: version.id, state: 'ready' }]
|
||||||
});
|
});
|
||||||
verified(id, version);
|
verified(id, version);
|
||||||
@@ -212,7 +212,7 @@ describe('automatic host preparation', () => {
|
|||||||
const t5 = await publish('t5');
|
const t5 = await publish('t5');
|
||||||
const t6 = await publish('t6');
|
const t6 = await publish('t6');
|
||||||
const readyInventory = {
|
const readyInventory = {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
|
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
|
||||||
};
|
};
|
||||||
const id = enroll(readyInventory);
|
const id = enroll(readyInventory);
|
||||||
@@ -310,7 +310,11 @@ describe('automatic host preparation', () => {
|
|||||||
expect(operations(host.id)).toHaveLength(1);
|
expect(operations(host.id)).toHaveLength(1);
|
||||||
|
|
||||||
vi.setSystemTime(Date.now() + 1);
|
vi.setSystemTime(Date.now() + 1);
|
||||||
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
const preparedInventory = {
|
||||||
|
...inventory,
|
||||||
|
preparedRuntimeImages: [runtime],
|
||||||
|
libraryCaches: []
|
||||||
|
};
|
||||||
const runtimeHeartbeat = await reportHeartbeat({
|
const runtimeHeartbeat = await reportHeartbeat({
|
||||||
request: request('heartbeat', { inventory: preparedInventory })
|
request: request('heartbeat', { inventory: preparedInventory })
|
||||||
});
|
});
|
||||||
@@ -369,7 +373,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({ runtimeImages: ['another:runtime'] });
|
const id = enroll({ preparedRuntimeImages: ['another:runtime'] });
|
||||||
const host = hosts.getHost(id)!;
|
const host = hosts.getHost(id)!;
|
||||||
const status = preparation.hostPreparationStatus(host);
|
const status = preparation.hostPreparationStatus(host);
|
||||||
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
|
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
|
||||||
@@ -406,7 +410,7 @@ describe('automatic host preparation', () => {
|
|||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)).toHaveLength(1);
|
expect(operations(id)).toHaveLength(1);
|
||||||
|
|
||||||
heartbeat(id, { runtimeImages: [runtime] });
|
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)).toHaveLength(2);
|
expect(operations(id)).toHaveLength(2);
|
||||||
const download = operations(id)[0];
|
const download = operations(id)[0];
|
||||||
@@ -417,7 +421,7 @@ describe('automatic host preparation', () => {
|
|||||||
verified(id, version);
|
verified(id, version);
|
||||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||||
heartbeat(id, {
|
heartbeat(id, {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: version.id, state: 'ready' }]
|
libraries: [{ id: version.id, state: 'ready' }]
|
||||||
});
|
});
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||||
@@ -432,7 +436,7 @@ describe('automatic host preparation', () => {
|
|||||||
game: 't5',
|
game: 't5',
|
||||||
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
||||||
});
|
});
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
const expected = new Map([latest, t6].map((version) => [version.id, version]));
|
const expected = new Map([latest, t6].map((version) => [version.id, version]));
|
||||||
const downloaded: string[] = [];
|
const downloaded: string[] = [];
|
||||||
for (let index = 0; index < expected.size; index++) {
|
for (let index = 0; index < expected.size; index++) {
|
||||||
@@ -452,7 +456,7 @@ describe('automatic host preparation', () => {
|
|||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)).toHaveLength(index + 1);
|
expect(operations(id)).toHaveLength(index + 1);
|
||||||
heartbeat(id, {
|
heartbeat(id, {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
|
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -462,12 +466,15 @@ describe('automatic host preparation', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)).toEqual([]);
|
expect(operations(id)).toEqual([]);
|
||||||
const old = await publish();
|
const old = await publish();
|
||||||
verified(id, old);
|
verified(id, old);
|
||||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
heartbeat(id, {
|
||||||
|
preparedRuntimeImages: [runtime],
|
||||||
|
libraries: [{ id: old.id, state: 'ready' }]
|
||||||
|
});
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||||
const latest = await publish();
|
const latest = await publish();
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
@@ -479,7 +486,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
||||||
const old = await publish();
|
const old = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
const first = operations(id)[0];
|
const first = operations(id)[0];
|
||||||
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
||||||
@@ -503,8 +510,8 @@ describe('automatic host preparation', () => {
|
|||||||
async (automatic) => {
|
async (automatic) => {
|
||||||
const old = await publish();
|
const old = await publish();
|
||||||
const image = 'ghcr.io/example/plutainer:queued-version';
|
const image = 'ghcr.io/example/plutainer:queued-version';
|
||||||
const id = enroll({ runtimeImages: [runtime, image] });
|
const id = enroll({ preparedRuntimeImages: [runtime, image] });
|
||||||
const other = enroll({ runtimeImages: [runtime] });
|
const other = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
const { queueNativeCreation } =
|
const { queueNativeCreation } =
|
||||||
await import('../../src/lib/server/provisioning/native.js');
|
await import('../../src/lib/server/provisioning/native.js');
|
||||||
for (let index = 0; index < 2; index++) {
|
for (let index = 0; index < 2; index++) {
|
||||||
@@ -521,7 +528,7 @@ describe('automatic host preparation', () => {
|
|||||||
).toBe(true);
|
).toBe(true);
|
||||||
}
|
}
|
||||||
// Also cover pre-existing intents whose image is no longer cached.
|
// Also cover pre-existing intents whose image is no longer cached.
|
||||||
heartbeat(id, { runtimeImages: [runtime] });
|
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||||
const latest = await publish();
|
const latest = await publish();
|
||||||
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
||||||
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
|
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
|
||||||
@@ -555,7 +562,7 @@ describe('automatic host preparation', () => {
|
|||||||
vi.setSystemTime(Date.now() + 1);
|
vi.setSystemTime(Date.now() + 1);
|
||||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||||
heartbeat(id, {
|
heartbeat(id, {
|
||||||
runtimeImages: [runtime, image],
|
preparedRuntimeImages: [runtime, image],
|
||||||
libraries: [{ id: old.id, state: 'ready' }]
|
libraries: [{ id: old.id, state: 'ready' }]
|
||||||
});
|
});
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
@@ -568,7 +575,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
||||||
const old = await publish();
|
const old = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
const download = operations(id)[0];
|
const download = operations(id)[0];
|
||||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||||
@@ -604,7 +611,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
||||||
const old = await publish();
|
const old = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
const [first] = hosts.pendingHostOperations(id);
|
const [first] = hosts.pendingHostOperations(id);
|
||||||
const latest = await publish();
|
const latest = await publish();
|
||||||
@@ -613,7 +620,10 @@ describe('automatic host preparation', () => {
|
|||||||
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
||||||
verified(id, old);
|
verified(id, old);
|
||||||
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
||||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
heartbeat(id, {
|
||||||
|
preparedRuntimeImages: [runtime],
|
||||||
|
libraries: [{ id: old.id, state: 'ready' }]
|
||||||
|
});
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)[0]).toMatchObject({
|
expect(operations(id)[0]).toMatchObject({
|
||||||
state: 'pending',
|
state: 'pending',
|
||||||
@@ -688,23 +698,26 @@ describe('automatic host preparation', () => {
|
|||||||
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({
|
const id = enroll({
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||||
});
|
});
|
||||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||||
expect(operations(id)).toEqual([]);
|
expect(operations(id)).toEqual([]);
|
||||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
heartbeat(id, {
|
||||||
|
preparedRuntimeImages: [runtime],
|
||||||
|
storageAvailable: 1_000 + version.sizeBytes
|
||||||
|
});
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
const operation = libraries.queueLibraryDownload(id, version.id);
|
const operation = libraries.queueLibraryDownload(id, version.id);
|
||||||
heartbeat(id, {
|
heartbeat(id, {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
|
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
|
||||||
});
|
});
|
||||||
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
||||||
@@ -715,7 +728,7 @@ describe('automatic host preparation', () => {
|
|||||||
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
||||||
const oversized = await publish();
|
const oversized = await publish();
|
||||||
const small = await publish('t6', 'small');
|
const small = await publish('t6', 'small');
|
||||||
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 });
|
||||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||||
const status = preparation.reconcileHostPreparation(id);
|
const status = preparation.reconcileHostPreparation(id);
|
||||||
expect(status.items).toEqual(
|
expect(status.items).toEqual(
|
||||||
@@ -731,7 +744,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
libraries.updateHostLibraryCaches(id, [
|
libraries.updateHostLibraryCaches(id, [
|
||||||
{
|
{
|
||||||
libraryVersionId: version.id,
|
libraryVersionId: version.id,
|
||||||
@@ -741,7 +754,7 @@ describe('automatic host preparation', () => {
|
|||||||
}
|
}
|
||||||
]);
|
]);
|
||||||
heartbeat(id, {
|
heartbeat(id, {
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
|
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
|
||||||
});
|
});
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||||
@@ -753,7 +766,7 @@ describe('automatic host preparation', () => {
|
|||||||
|
|
||||||
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({ runtimeImages: [runtime] });
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
const operation = operations(id)[0];
|
const operation = operations(id)[0];
|
||||||
libraries.updateHostLibraryCaches(id, [
|
libraries.updateHostLibraryCaches(id, [
|
||||||
@@ -768,11 +781,11 @@ describe('automatic host preparation', () => {
|
|||||||
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
||||||
const failedAt = Date.now();
|
const failedAt = Date.now();
|
||||||
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
||||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)).toHaveLength(1);
|
expect(operations(id)).toHaveLength(1);
|
||||||
vi.setSystemTime(failedAt + 60_000);
|
vi.setSystemTime(failedAt + 60_000);
|
||||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||||
expect(operations(id)).toHaveLength(2);
|
expect(operations(id)).toHaveLength(2);
|
||||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||||
@@ -800,7 +813,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
(state) => {
|
(state) => {
|
||||||
const ready = {
|
const ready = {
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||||
};
|
};
|
||||||
const id = enroll(ready);
|
const id = enroll(ready);
|
||||||
@@ -819,7 +832,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
||||||
const id = enroll({
|
const id = enroll({
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: 'game-files', state: 'ready' }]
|
libraries: [{ id: 'game-files', state: 'ready' }]
|
||||||
});
|
});
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
@@ -842,7 +855,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
const ready = {
|
const ready = {
|
||||||
...inventory,
|
...inventory,
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||||
};
|
};
|
||||||
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
||||||
@@ -903,7 +916,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
const version = await publish();
|
const version = await publish();
|
||||||
const ready = {
|
const ready = {
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: version.id, state: 'ready' as const }],
|
libraries: [{ id: version.id, state: 'ready' as const }],
|
||||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||||
};
|
};
|
||||||
@@ -948,7 +961,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
const pull = operations(id)[0];
|
const pull = operations(id)[0];
|
||||||
expect(pull.kind).toBe('runtime.pull');
|
expect(pull.kind).toBe('runtime.pull');
|
||||||
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
||||||
heartbeat(id, { ...capable, runtimeImages: [runtime] });
|
heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
const download = operations(id)[0];
|
const download = operations(id)[0];
|
||||||
expect(download.kind).toBe('library.download');
|
expect(download.kind).toBe('library.download');
|
||||||
@@ -956,7 +969,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||||
const ready = {
|
const ready = {
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||||
};
|
};
|
||||||
heartbeat(id, ready);
|
heartbeat(id, ready);
|
||||||
@@ -1005,7 +1018,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
||||||
const ready = {
|
const ready = {
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
plutoniumCache: { revision: 5354, bytes: 25_000 }
|
plutoniumCache: { revision: 5354, bytes: 25_000 }
|
||||||
};
|
};
|
||||||
const id = enroll(ready);
|
const id = enroll(ready);
|
||||||
@@ -1048,7 +1061,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
|
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
|
||||||
});
|
});
|
||||||
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
||||||
const id = enroll({ ...capable, runtimeImages: [runtime] });
|
const id = enroll({ ...capable, preparedRuntimeImages: [runtime] });
|
||||||
preparation.reconcileHostPreparation(id);
|
preparation.reconcileHostPreparation(id);
|
||||||
expect(operations(id)[0]).toMatchObject({
|
expect(operations(id)[0]).toMatchObject({
|
||||||
kind: 'plutonium.prepare',
|
kind: 'plutonium.prepare',
|
||||||
@@ -1060,7 +1073,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
).toBe('Latest production revision');
|
).toBe('Latest production revision');
|
||||||
const cached = enroll({
|
const cached = enroll({
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
plutoniumCache: { revision: 5353, bytes: 200 }
|
plutoniumCache: { revision: 5353, bytes: 200 }
|
||||||
});
|
});
|
||||||
preparation.reconcileHostPreparation(cached);
|
preparation.reconcileHostPreparation(cached);
|
||||||
@@ -1071,7 +1084,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
it('leaves agents without the cache capability unchanged', async () => {
|
it('leaves agents without the cache capability unchanged', async () => {
|
||||||
await release(5355);
|
await release(5355);
|
||||||
const id = enroll({
|
const id = enroll({
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||||
});
|
});
|
||||||
expect(
|
expect(
|
||||||
@@ -1084,7 +1097,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
const version = await publish();
|
const version = await publish();
|
||||||
const ready = {
|
const ready = {
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||||
};
|
};
|
||||||
const id = enroll(ready);
|
const id = enroll(ready);
|
||||||
@@ -1135,7 +1148,7 @@ describe('optional host Plutonium cache', () => {
|
|||||||
const version = await publish();
|
const version = await publish();
|
||||||
const id = enroll({
|
const id = enroll({
|
||||||
...capable,
|
...capable,
|
||||||
runtimeImages: [runtime],
|
preparedRuntimeImages: [runtime],
|
||||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||||
});
|
});
|
||||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ const inventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: ['plutainer:test'],
|
preparedRuntimeImages: ['plutainer:test'],
|
||||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||||
};
|
};
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ describe('native creation intents before placement', () => {
|
|||||||
memoryTotal: 32 * 1024 ** 3,
|
memoryTotal: 32 * 1024 ** 3,
|
||||||
memoryAvailable: 30 * 1024 ** 3,
|
memoryAvailable: 30 * 1024 ** 3,
|
||||||
storageAvailable: 100_000_000_000,
|
storageAvailable: 100_000_000_000,
|
||||||
runtimeImages: [image]
|
preparedRuntimeImages: [image]
|
||||||
};
|
};
|
||||||
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
|
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
|
||||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ const inventory = {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * 1024 ** 3,
|
storageAvailable: 100 * 1024 ** 3,
|
||||||
runtimeImages: ['plutainer:test'],
|
preparedRuntimeImages: ['plutainer:test'],
|
||||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||||
};
|
};
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
@@ -494,6 +494,21 @@ describe('native host enrollment and scoped authentication', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('native host readiness and reservations', () => {
|
describe('native host readiness and reservations', () => {
|
||||||
|
it.each([
|
||||||
|
{ preparedRuntimeImages: 'plutainer:test' },
|
||||||
|
{ preparedRuntimeImages: [123] },
|
||||||
|
{ preparedRuntimeImages: null }
|
||||||
|
])(
|
||||||
|
'rejects malformed prepared runtime inventory: $preparedRuntimeImages',
|
||||||
|
async ({ preparedRuntimeImages }) => {
|
||||||
|
const api = await enrolled();
|
||||||
|
expect(() =>
|
||||||
|
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages })
|
||||||
|
).toThrow(/runtime image inventory/i);
|
||||||
|
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
it('separates a connected agent from ready Docker, storage, runtime and library state', async () => {
|
it('separates a connected agent from ready Docker, storage, runtime and library state', async () => {
|
||||||
const api = await enrolled();
|
const api = await enrolled();
|
||||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||||
@@ -518,8 +533,8 @@ describe('native host readiness and reservations', () => {
|
|||||||
});
|
});
|
||||||
it.each([
|
it.each([
|
||||||
{ missing: { libraries: [] }, expected: 'needs-game-files' },
|
{ missing: { libraries: [] }, expected: 'needs-game-files' },
|
||||||
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
|
{ missing: { preparedRuntimeImages: [] }, expected: 'needs-runtime' },
|
||||||
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
{ missing: { preparedRuntimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||||
])(
|
])(
|
||||||
'names missing content as $expected until a prepared heartbeat arrives',
|
'names missing content as $expected until a prepared heartbeat arrives',
|
||||||
async ({ missing, expected }) => {
|
async ({ missing, expected }) => {
|
||||||
@@ -545,7 +560,7 @@ describe('native host readiness and reservations', () => {
|
|||||||
download.id,
|
download.id,
|
||||||
runtime.id
|
runtime.id
|
||||||
]);
|
]);
|
||||||
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
|
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages: [], libraries: [] });
|
||||||
expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([
|
expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([
|
||||||
download.id,
|
download.id,
|
||||||
runtime.id
|
runtime.id
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 1_000_000_000_000,
|
storageAvailable: 1_000_000_000_000,
|
||||||
runtimeImages: [image]
|
preparedRuntimeImages: [image]
|
||||||
};
|
};
|
||||||
enrollHost(createEnrollment(hostId).token, inventory);
|
enrollHost(createEnrollment(hostId).token, inventory);
|
||||||
const { createLibraryImport, updateHostLibraryCaches } =
|
const { createLibraryImport, updateHostLibraryCaches } =
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ const inventory = () => ({
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100_000_000_000,
|
storageAvailable: 100_000_000_000,
|
||||||
runtimeImages: [image],
|
preparedRuntimeImages: [image],
|
||||||
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
|
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
|
||||||
});
|
});
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100_000_000_000,
|
storageAvailable: 100_000_000_000,
|
||||||
runtimeImages: ['plutainer:test'],
|
preparedRuntimeImages: ['plutainer:test'],
|
||||||
libraries: [{ id: 'published-game-version', state: 'ready' }],
|
libraries: [{ id: 'published-game-version', state: 'ready' }],
|
||||||
capabilities: ['instance.safety-override']
|
capabilities: ['instance.safety-override']
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ async function heartbeat(hostId: string, cpuUsagePercent: number) {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * gib,
|
storageAvailable: 100 * gib,
|
||||||
runtimeImages: [image],
|
preparedRuntimeImages: [image],
|
||||||
libraries: [{ id: libraryId, state: 'ready' }],
|
libraries: [{ id: libraryId, state: 'ready' }],
|
||||||
instanceResources: instances.map((instance) => ({
|
instanceResources: instances.map((instance) => ({
|
||||||
instanceId: instance.id,
|
instanceId: instance.id,
|
||||||
@@ -92,7 +92,7 @@ beforeEach(async () => {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100 * gib,
|
storageAvailable: 100 * gib,
|
||||||
runtimeImages: [image],
|
preparedRuntimeImages: [image],
|
||||||
libraries: [{ id: libraryId, state: 'ready' }]
|
libraries: [{ id: libraryId, state: 'ready' }]
|
||||||
});
|
});
|
||||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||||
|
|||||||
@@ -548,7 +548,7 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
|||||||
dockerRunning: true,
|
dockerRunning: true,
|
||||||
directoriesWritable: true,
|
directoriesWritable: true,
|
||||||
storageAvailable: 100_000_000_000,
|
storageAvailable: 100_000_000_000,
|
||||||
runtimeImages: [config.defaultRuntimeImage],
|
preparedRuntimeImages: [config.defaultRuntimeImage],
|
||||||
libraries: [{ id: 'lib', state: 'ready' }]
|
libraries: [{ id: 'lib', state: 'ready' }]
|
||||||
});
|
});
|
||||||
const generation = action === 'create' ? 1 : 2;
|
const generation = action === 'create' ? 1 : 2;
|
||||||
|
|||||||
@@ -23,16 +23,31 @@ afterEach(async () => {
|
|||||||
await fixture.cleanup();
|
await fixture.cleanup();
|
||||||
});
|
});
|
||||||
|
|
||||||
async function cache(images: string[]) {
|
async function reportImages(
|
||||||
|
preparedImages: string[] | undefined,
|
||||||
|
cachedImages = preparedImages ?? []
|
||||||
|
) {
|
||||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||||
const host = hosts.getHost(prepared.host.id)!;
|
const host = hosts.getHost(prepared.host.id)!;
|
||||||
hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images });
|
hosts.heartbeat(host.id, {
|
||||||
|
...host.inventory,
|
||||||
|
runtimeImages: cachedImages,
|
||||||
|
preparedRuntimeImages: preparedImages
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
describe('runtime image launch policy', () => {
|
describe('runtime image launch policy', () => {
|
||||||
it.each([false, true])(
|
it.each([
|
||||||
'rejects an uncached custom image at POST /api/servers (automatic: %s)',
|
{ automatic: false, cached: false, legacy: false },
|
||||||
async (automatic) => {
|
{ automatic: true, cached: false, legacy: false },
|
||||||
|
{ automatic: false, cached: true, legacy: false },
|
||||||
|
{ automatic: true, cached: true, legacy: false },
|
||||||
|
{ automatic: false, cached: true, legacy: true },
|
||||||
|
{ automatic: true, cached: true, legacy: true }
|
||||||
|
])(
|
||||||
|
'rejects an unprepared custom image at POST /api/servers ($automatic, cached: $cached, legacy: $legacy)',
|
||||||
|
async ({ automatic, cached, legacy }) => {
|
||||||
|
await reportImages(legacy ? undefined : [], cached ? [custom] : []);
|
||||||
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
||||||
const response = await POST({
|
const response = await POST({
|
||||||
request: new Request('https://controller.test/api/servers', {
|
request: new Request('https://controller.test/api/servers', {
|
||||||
@@ -57,13 +72,88 @@ describe('runtime image launch policy', () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
it('allows the configured default before it is cached', async () => {
|
it('allows the configured default before it is cached', async () => {
|
||||||
await cache([]);
|
await reportImages([]);
|
||||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||||
expect(queueNativeCreation(launch).ok).toBe(true);
|
expect(queueNativeCreation(launch).ok).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps the cached configured default eligible with a legacy agent', async () => {
|
||||||
|
const { config } = await import('../../src/lib/server/config.js');
|
||||||
|
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||||
|
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
||||||
|
expect(nativeSelection({}, 't5').hostId).toBe(prepared.host.id);
|
||||||
|
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||||
|
expect(getHost(prepared.host.id)!.inventory.preparedRuntimeImages).toEqual([
|
||||||
|
config.defaultRuntimeImage
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores malformed preparation fields already stored by an older controller', async () => {
|
||||||
|
const { config } = await import('../../src/lib/server/config.js');
|
||||||
|
const { db } = await import('../../src/lib/server/db/index.js');
|
||||||
|
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||||
|
const inventory = getHost(prepared.host.id)!.inventory;
|
||||||
|
for (const reported of [{}, 42, custom]) {
|
||||||
|
db()
|
||||||
|
.prepare('UPDATE ec_hosts SET inventory=? WHERE id=?')
|
||||||
|
.run(
|
||||||
|
JSON.stringify({
|
||||||
|
...inventory,
|
||||||
|
runtimeImages: [config.defaultRuntimeImage, custom],
|
||||||
|
preparedRuntimeImages: reported
|
||||||
|
}),
|
||||||
|
prepared.host.id
|
||||||
|
);
|
||||||
|
expect(getHost(prepared.host.id)).toMatchObject({
|
||||||
|
readiness: 'ready',
|
||||||
|
inventory: { preparedRuntimeImages: [config.defaultRuntimeImage] }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('hides unrelated cached images from host list, detail, and page responses', async () => {
|
||||||
|
const { config } = await import('../../src/lib/server/config.js');
|
||||||
|
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||||
|
const list = await import('../../src/routes/api/hosts/+server.js');
|
||||||
|
const detail = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||||
|
const { load } = await import('../../src/routes/hosts/+page.server.js');
|
||||||
|
const request = new Request('https://controller.test/api/hosts');
|
||||||
|
const bodies = [
|
||||||
|
await list.GET({ request }).json(),
|
||||||
|
await detail.GET({ request, params: { id: prepared.host.id } }).json(),
|
||||||
|
load()
|
||||||
|
];
|
||||||
|
for (const body of bodies) {
|
||||||
|
const host = 'hosts' in body ? body.hosts[0] : body.host;
|
||||||
|
expect(host.inventory.runtimeImages).toBeUndefined();
|
||||||
|
expect(host.inventory.preparedRuntimeImages).toEqual([config.defaultRuntimeImage]);
|
||||||
|
expect(JSON.stringify(body)).not.toContain(custom);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not place an accepted runtime on another host that only caches it', async () => {
|
||||||
|
await reportImages([custom]);
|
||||||
|
const other = await prepareNativeHost();
|
||||||
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||||
|
hosts.heartbeat(other.host.id, {
|
||||||
|
...hosts.getHost(other.host.id)!.inventory,
|
||||||
|
runtimeImages: [custom]
|
||||||
|
});
|
||||||
|
expect(() => hosts.requireReadyHost(other.host.id, { image: custom })).toThrow(
|
||||||
|
/runtime image is not prepared/
|
||||||
|
);
|
||||||
|
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
||||||
|
expect(() =>
|
||||||
|
choosePlacement({
|
||||||
|
hostId: other.host.id,
|
||||||
|
libraryId: other.versions.get('t5')!,
|
||||||
|
image: custom
|
||||||
|
})
|
||||||
|
).toThrow(/runtime image is not prepared/);
|
||||||
|
});
|
||||||
|
|
||||||
it('accepts prepared images only on hosts that have them', async () => {
|
it('accepts prepared images only on hosts that have them', async () => {
|
||||||
await cache([custom]);
|
await reportImages([custom]);
|
||||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||||
const other = hosts.createHost({ name: 'Unprepared host' });
|
const other = hosts.createHost({ name: 'Unprepared host' });
|
||||||
const { nativeLaunchSelection, nativeSelection } =
|
const { nativeLaunchSelection, nativeSelection } =
|
||||||
@@ -81,7 +171,7 @@ describe('runtime image launch policy', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('requires exact image references and does not treat a queued admin pull as cached', async () => {
|
it('requires exact prepared references and does not treat a queued admin pull as preparation', async () => {
|
||||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||||
hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom });
|
hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom });
|
||||||
const { nativeLaunchSelection } =
|
const { nativeLaunchSelection } =
|
||||||
@@ -89,25 +179,25 @@ describe('runtime image launch policy', () => {
|
|||||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||||
/prepared by an admin/
|
/prepared by an admin/
|
||||||
);
|
);
|
||||||
await cache([custom + '-other']);
|
await reportImages([custom + '-other']);
|
||||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||||
/prepared by an admin/
|
/prepared by an admin/
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rechecks cached images when submitting a saved single-server preview', async () => {
|
it('rechecks preparation when submitting a saved preview even if the image is still cached', async () => {
|
||||||
await cache([custom]);
|
await reportImages([custom]);
|
||||||
const { previewNativeCreation, queueNativeCreation } =
|
const { previewNativeCreation, queueNativeCreation } =
|
||||||
await import('../../src/lib/server/provisioning/native.js');
|
await import('../../src/lib/server/provisioning/native.js');
|
||||||
const preview = previewNativeCreation({ ...launch, image: custom });
|
const preview = previewNativeCreation({ ...launch, image: custom });
|
||||||
await cache([]);
|
await reportImages([], [custom]);
|
||||||
expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({
|
expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({
|
||||||
ok: false,
|
ok: false,
|
||||||
message: expect.stringContaining('prepared by an admin')
|
message: expect.stringContaining('prepared by an admin')
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => {
|
it('rejects unprepared collection images and rechecks saved previews before writing any jobs', async () => {
|
||||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||||
const { previewCollection, queueCollection } =
|
const { previewCollection, queueCollection } =
|
||||||
await import('../../src/lib/server/fleet/collections.js');
|
await import('../../src/lib/server/fleet/collections.js');
|
||||||
@@ -115,15 +205,16 @@ describe('runtime image launch policy', () => {
|
|||||||
name: 'Image policy',
|
name: 'Image policy',
|
||||||
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
||||||
});
|
});
|
||||||
|
await reportImages([], [custom]);
|
||||||
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(
|
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(
|
||||||
/prepared by an admin/
|
/prepared by an admin/
|
||||||
);
|
);
|
||||||
await cache([custom]);
|
await reportImages([custom]);
|
||||||
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
|
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
|
||||||
const keys = Object.fromEntries(
|
const keys = Object.fromEntries(
|
||||||
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
||||||
);
|
);
|
||||||
await cache([]);
|
await reportImages([], [custom]);
|
||||||
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(
|
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(
|
||||||
/prepared by an admin/
|
/prepared by an admin/
|
||||||
);
|
);
|
||||||
@@ -136,7 +227,7 @@ describe('runtime image launch policy', () => {
|
|||||||
]) {
|
]) {
|
||||||
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||||
}
|
}
|
||||||
await cache([custom]);
|
await reportImages([custom]);
|
||||||
expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true);
|
expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user