diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 68dd1ff..a9b2abd 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.5.0" +VERSION = "1.5.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1505,7 +1505,9 @@ class Agent: atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) name = self.container_name(instance_id, generation) spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) - args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] + # Do not let Docker fetch a replacement if the prepared image disappears + # between validation and container creation. + args = ["container", "create", "--pull=never", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] if self.config.get("hostId"): args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])]) bind_arg = "[" + bind + "]" if ":" in bind else bind diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 9bc1c64..a32e2e3 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -727,6 +727,29 @@ class AgentTests(unittest.TestCase): self.assertEqual(second["result"], first["result"]) self.assertEqual(sum(call[:2] == ("container", "restart") for call in self.docker.calls), 1) + def test_creation_uses_cached_runtime_without_allowing_docker_to_pull(self): + payload = self.create() + creation = next(args for args in self.docker.calls if args[:2] == ("container", "create")) + self.assertIn("--pull=never", creation) + self.assertEqual(creation[-1], payload["image"]) + self.assertFalse(any(args[:2] == ("image", "pull") for args in self.docker.calls)) + + def test_image_disappearing_after_validation_cannot_be_pulled_or_started(self): + original = self.docker.call + + def missing_image(*args, **kwargs): + if args[:2] == ("container", "create"): + self.assertIn("--pull=never", args) + raise AgentError("No such image") + return original(*args, **kwargs) + + payload = {"game": "t5", "libraryId": self.library_id, "image": "prepared:runtime", "port": 28960} + with patch.object(self.docker, "call", side_effect=missing_image): + with self.assertRaisesRegex(AgentError, "No such image"): + self.agent.create(INSTANCE, 1, payload) + self.assertEqual(self.docker.started, 0) + self.assertFalse(any(args[:2] == ("image", "pull") for args in self.docker.calls)) + def test_manifest_is_advertised_and_reconciles_as_a_read_operation(self): self.create() operation = self.operation("files.manifest", {"roots": ["configs", "scripts"], "ignoreNames": [], "paths": ["configs/server.cfg"]}) diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 6c23a4b..7af07a0 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -35,7 +35,8 @@ Status replies still expire normally, and native timeout diagnostics are retaine game files, image, capacity and ports required for the launch. 6. Launch a server or collection. The latest game files and default runtime are selected automatically. An advanced runtime override is - available. If the latest files change after previewing, refresh the preview + available for images an admin has already prepared on the destination host. + If the latest files change after previewing, refresh the preview before launching. Apply saves the requested lobby IDs. Provisioning selects a host and atomically reserves UDP/TCP ports when capacity is available. @@ -153,8 +154,12 @@ memory measurements. Hosts with stale or missing measurements cannot accept new servers. Existing API clients may still explicitly constrain a launch to a host. Queued jobs retain their accepted game-file version. Automatic host preparation -also downloads the files and runtime images needed by unplaced jobs; a newer -upload does not remove files still needed by the queue. +also downloads the game files needed by unplaced jobs; a newer upload does not +remove files still needed by the queue. Only the configured default runtime is +pulled automatically. Custom images must already be cached on the selected host, +or on an active host for automatic placement, before a launch is accepted. Admins +can prepare them from the host page. Queued launches and migrations never trigger +custom image pulls, and container creation cannot implicitly pull a missing image. Only provisioning reserves a host and port. Agent **1.4.6** fixes container discovery for resource sampling (the diff --git a/src/lib/server/hosts/preparation.ts b/src/lib/server/hosts/preparation.ts index a501222..8c6da02 100644 --- a/src/lib/server/hosts/preparation.ts +++ b/src/lib/server/hosts/preparation.ts @@ -10,22 +10,21 @@ const activeStates = ['pending', 'accepted', 'running', 'unknown']; const supersededDownload = 'Superseded by newer game files before delivery to the host.'; type PreparationOperation = { id: string; sequence: number; last_delivered_at: string | null; state: HostOperationState; updated_at: string; error: string | null }; type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation }; -type QueuedSelection = { libraryId: string; image: string | null }; +type QueuedSelection = { libraryId: string }; type LibraryTarget = Pick[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean }; /** Accepted requests retain their prerequisites until placement owns an instance. */ function queuedSelections(hostId: string): QueuedSelection[] { // Standalone host/library readers can initialize before provisioning migrations. if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return []; - return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId, - json_extract(q.selection,'$.image') AS image FROM native_creation_intents q + return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId FROM native_creation_intents q JOIN server_deployments d ON d.id=q.deployment_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL OR json_extract(q.selection,'$.hostId')=?) - UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m + UNION SELECT i.library_id AS libraryId FROM ec_host_migrations m JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') @@ -88,8 +87,8 @@ function targets(host: Host): Target[] { && operation.sequence > enabling.operation_floor && operation.state === 'completed' && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at); const selections = queuedSelections(host.id); - const images = new Set([config.defaultRuntimeImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]); - const result: Target[] = [...images].map(image => { + // Queued launches and migrations are not authority to pull custom images. + const result: Target[] = [config.defaultRuntimeImage].map(image => { const operation = latestOperation(host.id, 'runtime.pull', 'image', image); const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); return { image, requiredBytes: host.minimumFreeBytes, diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 1c84319..781d8fb 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -9,7 +9,7 @@ import { ensureHostSchema } from '../hosts/schema.js'; import { unsettledCancelledCommand } from './cancellation.js'; import { config, relDestForKind, serverDir } from '../config.js'; import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js'; -import { getHost, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; +import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js'; import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js'; @@ -55,6 +55,14 @@ export function nativeLaunchSelection(input: Record, game: stri if (!host) throw new Error('The selected host was not found.'); if (host.decommission) throw new Error('The selected host is being removed. Choose another host.'); } + // Only the controller-configured default may be prepared automatically. + // Custom runtimes must already be present in trusted host inventory; a + // launch request must never authorize a registry download. + if (image !== config.defaultRuntimeImage && !listHosts().some(host => + !host.decommission && (automatic ? host.mode === 'active' : host.id === input.hostId) + && host.inventory.runtimeImages?.includes(image))) { + throw new Error('Custom runtime images must be prepared by an admin from the host page before launching.'); + } const rawNetwork = input.automatic === true ? input.network ?? {} : input; if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.'); const network = rawNetwork as NativeSelection; diff --git a/src/lib/ui/components/NativeHostSelection.svelte b/src/lib/ui/components/NativeHostSelection.svelte index c1285ce..0949e81 100644 --- a/src/lib/ui/components/NativeHostSelection.svelte +++ b/src/lib/ui/components/NativeHostSelection.svelte @@ -57,7 +57,7 @@ {/each}{/if}

Hosts keep the latest T5 and T6 files ready automatically. Upload game files when they change. Existing servers keep their current files.

-
Advanced: runtime override

The configured default runtime is prepared automatically. Custom images must be prepared from the host page.

+
Advanced: runtime override

The configured default runtime is prepared automatically. Custom images must first be prepared by an admin from the destination host page.

{#if network}
Network overrides

Public mapping does not change the local Docker port reservation.

{/if} diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 68dd1ff..a9b2abd 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.5.0" +VERSION = "1.5.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1505,7 +1505,9 @@ class Agent: atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) name = self.container_name(instance_id, generation) spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) - args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] + # Do not let Docker fetch a replacement if the prepared image disappears + # between validation and container creation. + args = ["container", "create", "--pull=never", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] if self.config.get("hostId"): args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])]) bind_arg = "[" + bind + "]" if ":" in bind else bind diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 7a6150d..4772aec 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.5.0", + "version": "1.5.1", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "ea0455b5439eb51ef86f9abbe5e7bb703ad5aa23397377e2f1e57a61fe56dcde", + "sha256": "67003980127327d07144041d0c17b8adaa0d466b30b0ebf7508a80efd5cf6481", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/host-preparation.test.ts b/tests/unit/host-preparation.test.ts index cc02006..ba26d9f 100644 --- a/tests/unit/host-preparation.test.ts +++ b/tests/unit/host-preparation.test.ts @@ -337,37 +337,35 @@ describe('automatic host preparation', () => { expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); }); - it.each([false, true])('prepares queued runtime and library versions after newer publication (automatic placement: %s)', async automatic => { + it.each([false, true])('prepares queued libraries without pulling custom runtimes (automatic placement: %s)', async automatic => { const old = await publish(); - const id = enroll({ runtimeImages: [runtime] }); - const other = enroll({ runtimeImages: [runtime] }); const image = 'ghcr.io/example/plutainer:queued-version'; + const id = enroll({ runtimeImages: [runtime, image] }); + const other = enroll({ runtimeImages: [runtime] }); const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); for (let index = 0; index < 2; index++) { expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id, hostId: automatic ? undefined : id, image, plutoKey: 'saved-key' }).ok).toBe(true); } + // Also cover pre-existing intents whose image is no longer cached. + heartbeat(id, { runtimeImages: [runtime] }); const latest = await publish(); const status = preparation.hostPreparationStatus(hosts.getHost(id)!); - expect(status.items.filter(item => item.label === image)).toHaveLength(1); + expect(status.items.filter(item => item.label === image)).toHaveLength(0); expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1); const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!); - expect(otherStatus.items.some(item => item.label === image)).toBe(automatic); + expect(otherStatus.items.some(item => item.label === image)).toBe(false); expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic); expect(operations(id)).toEqual([]); - preparation.reconcileHostPreparation(id); - const pull = operations(id)[0]; - expect(pull).toMatchObject({ kind: 'runtime.pull', payload: { image } }); - vi.setSystemTime(Date.now() + 1); - hosts.reportHostOperation(id, pull.id, { state: 'completed' }); - heartbeat(id, { runtimeImages: [runtime, image] }); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } }); preparation.reconcileHostPreparation(id); expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); - expect(operations(id)).toHaveLength(2); + expect(operations(id)).toHaveLength(1); + preparation.reconcileHostPreparation(other); + expect([...operations(id), ...operations(other)].every(operation => operation.kind !== 'runtime.pull')).toBe(true); verified(id, old); vi.setSystemTime(Date.now() + 1); diff --git a/tests/unit/native-creation-intents.test.ts b/tests/unit/native-creation-intents.test.ts index 5da7f77..0c0d782 100644 --- a/tests/unit/native-creation-intents.test.ts +++ b/tests/unit/native-creation-intents.test.ts @@ -120,6 +120,7 @@ describe('native creation intents before placement', () => { it('starts work on a ready host when an older pinned intent is waiting for an offline host', async () => { vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); const image = 'plutainer:test'; + vi.stubEnv('PLUTAINER_IMAGE', image); const hosts = await import('../../src/lib/server/hosts/index.js'); const offline = hosts.createHost({ name: 'Offline host' }); const ready = hosts.createHost({ name: 'Ready host' }); diff --git a/tests/unit/runtime-image-policy.test.ts b/tests/unit/runtime-image-policy.test.ts new file mode 100644 index 0000000..9e4c23b --- /dev/null +++ b/tests/unit/runtime-image-policy.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { prepareNativeHost } from '../helpers/native-host.js'; + +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +let fixture: Fixture; +let prepared: Awaited>; +const custom = 'registry.example/untrusted:latest'; +const launch = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' }; + +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); + prepared = await prepareNativeHost(); +}); +afterEach(async () => { await fixture.cleanup(); }); + +async function cache(images: string[]) { + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.getHost(prepared.host.id)!; + hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images }); +} + +describe('runtime image launch policy', () => { + it.each([false, true])('rejects an uncached custom image at POST /api/servers (automatic: %s)', async automatic => { + const { POST } = await import('../../src/routes/api/servers/+server.js'); + const response = await POST({ request: new Request('https://controller.test/api/servers', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...launch, image: custom, ...(automatic ? {} : { hostId: prepared.host.id }) }) + }) }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: expect.stringContaining('prepared by an admin') }); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const table of ['native_creation_intents', 'server_deployments', 'server_keys']) { + expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); + } + }); + + it('allows the configured default before it is cached', async () => { + await cache([]); + const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); + expect(queueNativeCreation(launch).ok).toBe(true); + }); + + it('accepts prepared images only on hosts that have them', async () => { + await cache([custom]); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const other = hosts.createHost({ name: 'Unprepared host' }); + const { nativeLaunchSelection, nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); + expect(nativeLaunchSelection({ hostId: prepared.host.id, image: custom }, 't5').image).toBe(custom); + expect(nativeSelection({ image: custom }, 't5').hostId).toBe(prepared.host.id); + expect(() => nativeLaunchSelection({ hostId: other.id, image: custom }, 't5')).toThrow(/prepared by an admin/); + hosts.updateHost(prepared.host.id, { mode: 'disabled' }); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + }); + + it('requires exact image references and does not treat a queued admin pull as cached', async () => { + const hosts = await import('../../src/lib/server/hosts/index.js'); + hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom }); + const { nativeLaunchSelection } = await import('../../src/lib/server/provisioning/native.js'); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + await cache([custom + '-other']); + expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(/prepared by an admin/); + }); + + it('rechecks cached images when submitting a saved single-server preview', async () => { + await cache([custom]); + const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); + const preview = previewNativeCreation({ ...launch, image: custom }); + await cache([]); + expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({ + ok: false, message: expect.stringContaining('prepared by an admin') + }); + }); + + it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => { + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ name: 'Image policy', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] }); + expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(/prepared by an admin/); + await cache([custom]); + const preview = previewCollection(blueprint.id, 'fill', { image: custom }); + const keys = Object.fromEntries(preview.servers.map(server => [server.name!, 'saved-key'])); + await cache([]); + expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/prepared by an admin/); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const table of ['server_launch_collections', 'native_creation_intents', 'server_deployments', 'server_keys']) { + expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); + } + await cache([custom]); + expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true); + }); +});