From 4b0f341d64e2877694e14a98bf16abdbb6888ac2 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 18:07:07 -0400 Subject: [PATCH] Remove Komodo backend in favor of native hosts --- .env.example | 2 - README.md | 4 - docs/deployment-blueprints.md | 11 +- docs/maintenance.md | 2 +- docs/native-hosts.md | 11 +- docs/provisioning-improvement-plan.md | 4 +- docs/provisioning.md | 36 +- docs/t5-t6-idle-cpu-research.md | 4 +- docs/ui-ux-implementation.md | 4 +- docs/ui-ux-review.md | 6 +- komodo/stack.toml | 41 - src/lib/provisioning-detail.ts | 8 +- src/lib/server/config.ts | 30 +- src/lib/server/db/index.ts | 11 +- src/lib/server/db/schema.sql | 7 +- src/lib/server/fleet/beta.ts | 2 +- src/lib/server/fleet/collections.ts | 65 +- src/lib/server/fleet/health-observations.ts | 19 +- src/lib/server/fleet/launch-recipes.ts | 2 +- src/lib/server/fleet/servers.ts | 14 +- src/lib/server/fleet/snapshot.ts | 8 +- src/lib/server/fleet/status-evidence.ts | 2 +- src/lib/server/fleet/status.ts | 3 +- src/lib/server/hosts/capacity-overview.ts | 2 +- src/lib/server/hosts/preparation.ts | 2 +- src/lib/server/integrations/komodo.ts | 832 --------------- src/lib/server/integrations/netProbe.ts | 93 -- src/lib/server/integrations/plutonium.ts | 4 +- src/lib/server/integrations/rcon.ts | 40 +- src/lib/server/maintenance/recovery.ts | 9 +- src/lib/server/pipeline/restart-rollout.ts | 2 +- src/lib/server/pipeline/worker.ts | 4 +- src/lib/server/provisioning/history.ts | 19 +- src/lib/server/provisioning/index.ts | 2 +- src/lib/server/provisioning/instances.ts | 965 +----------------- src/lib/server/provisioning/native.ts | 6 +- src/lib/server/provisioning/progress.ts | 2 +- src/lib/server/provisioning/requests.ts | 26 +- src/lib/server/provisioning/restarts.ts | 175 +--- src/lib/server/provisioning/worker.ts | 175 +--- .../ui/components/ProvisioningProgress.svelte | 2 +- src/lib/ui/komodoError.ts | 27 - src/lib/ui/provisioning-presentation.ts | 39 +- src/lib/ui/rconError.ts | 2 +- src/lib/ui/stores/fleet.ts | 5 +- src/routes/+page.svelte | 2 - src/routes/api/hosts/+server.ts | 2 +- src/routes/api/hosts/[id]/+server.ts | 2 +- .../[name]/diagnose-network/+server.ts | 77 -- .../servers/[name]/probe-komodo/+server.ts | 27 - src/routes/api/system-status/+server.ts | 55 +- src/routes/servers/[name]/+page.svelte | 228 +---- tests/helpers/agent-files.ts | 23 + tests/helpers/native-host.ts | 33 + tests/unit/beta-destruction.test.ts | 109 +- tests/unit/beta-plans.test.ts | 30 +- tests/unit/beta-promotion.test.ts | 2 +- tests/unit/beta-staging.test.ts | 2 +- .../collection-launch-performance.test.ts | 4 +- .../unit/collection-match-performance.test.ts | 8 +- tests/unit/config-flow.test.ts | 2 +- tests/unit/create-server.test.ts | 803 --------------- tests/unit/creation-queue.test.ts | 403 -------- tests/unit/deleted-server-guards.test.ts | 2 +- tests/unit/deployment-collections.test.ts | 410 +------- tests/unit/destruction-concurrency.test.ts | 215 ---- tests/unit/drift-kinds.test.ts | 2 +- tests/unit/fleet-monitor.test.ts | 2 +- tests/unit/fleet-overview.test.ts | 5 +- tests/unit/fleet-snapshot-scope.test.ts | 4 +- tests/unit/github-plugin-import.test.ts | 2 +- tests/unit/health-observations.test.ts | 41 +- tests/unit/host-instances.test.ts | 6 +- tests/unit/host-status-batch.test.ts | 10 +- tests/unit/komodo-health.test.ts | 106 -- tests/unit/komodo-runtime.test.ts | 36 - tests/unit/komodo-tags.test.ts | 88 -- tests/unit/komodo-write.test.ts | 29 - tests/unit/komodo.test.ts | 148 --- tests/unit/launch-recipes.test.ts | 255 +---- tests/unit/leaderboard-records.test.ts | 4 +- tests/unit/map-config-flow.test.ts | 2 +- tests/unit/match-detail.test.ts | 6 +- tests/unit/metrics.test.ts | 8 +- tests/unit/native-creation-intents.test.ts | 11 +- tests/unit/native-drift-manifest.test.ts | 4 +- tests/unit/native-launch-preparation.test.ts | 2 +- tests/unit/native-observations.test.ts | 2 +- tests/unit/native-provisioning.test.ts | 13 +- tests/unit/native-registry-migration.test.ts | 51 + tests/unit/native-safety.test.ts | 61 +- tests/unit/operation-history.test.ts | 4 +- tests/unit/plugin-versions.test.ts | 2 +- tests/unit/plutonium.test.ts | 258 +---- tests/unit/provisioning-cancellation.test.ts | 2 +- tests/unit/provisioning-detail.test.ts | 8 +- tests/unit/provisioning-evidence.test.ts | 405 -------- .../provisioning-history-performance.test.ts | 8 +- tests/unit/provisioning-presentation.test.ts | 13 +- tests/unit/provisioning-progress.test.ts | 6 +- tests/unit/provisioning.test.ts | 561 ---------- tests/unit/rcon-status.test.ts | 7 +- tests/unit/restart-rollout.test.ts | 494 --------- tests/unit/rollback-staging.test.ts | 4 +- tests/unit/script-flow.test.ts | 2 +- tests/unit/script-upload.test.ts | 2 +- tests/unit/server-chat.test.ts | 6 +- tests/unit/server-detail-api.test.ts | 4 +- tests/unit/server-history-storage.test.ts | 6 +- tests/unit/server-status.test.ts | 26 +- tests/unit/staged-files.test.ts | 356 +------ tests/unit/sweep.test.ts | 4 +- tests/unit/system-status.test.ts | 53 +- tests/unit/t6-config-flow.test.ts | 2 +- tests/unit/ui-server-clock.test.ts | 2 +- tests/unit/watchdog.test.ts | 388 +------ 116 files changed, 557 insertions(+), 8132 deletions(-) delete mode 100644 komodo/stack.toml delete mode 100644 src/lib/server/integrations/komodo.ts delete mode 100644 src/lib/server/integrations/netProbe.ts delete mode 100644 src/lib/ui/komodoError.ts delete mode 100644 src/routes/api/servers/[name]/diagnose-network/+server.ts delete mode 100644 src/routes/api/servers/[name]/probe-komodo/+server.ts create mode 100644 tests/helpers/agent-files.ts create mode 100644 tests/helpers/native-host.ts delete mode 100644 tests/unit/create-server.test.ts delete mode 100644 tests/unit/creation-queue.test.ts delete mode 100644 tests/unit/destruction-concurrency.test.ts delete mode 100644 tests/unit/komodo-health.test.ts delete mode 100644 tests/unit/komodo-runtime.test.ts delete mode 100644 tests/unit/komodo-tags.test.ts delete mode 100644 tests/unit/komodo-write.test.ts delete mode 100644 tests/unit/komodo.test.ts create mode 100644 tests/unit/native-registry-migration.test.ts delete mode 100644 tests/unit/provisioning-evidence.test.ts delete mode 100644 tests/unit/provisioning.test.ts delete mode 100644 tests/unit/restart-rollout.test.ts diff --git a/.env.example b/.env.example index 013a54d..7124076 100644 --- a/.env.example +++ b/.env.example @@ -28,8 +28,6 @@ PLUTAINER_IMAGE=ghcr.io/ayymoss/plutainer:latest # Browser login on localhost only (never for an exposed controller): # EC_AUTH_ALLOW_HTTP=true -# Existing legacy registrations can still use their previous KOMODO_*, -# SERVERS_ROOT, T6_SERVERS_ROOT, and RCON_HOST settings. Native hosts need none. # Fleet concurrency and campaign retention are saved in Settings, not env vars. # Optional request/report timings and event-loop delay diagnostics. diff --git a/README.md b/README.md index f34ca33..b3ae330 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ Electric Cherry manages T5 and T6 Plutonium Zombies servers through its own host agents. Docker and Plutainer run the games; EC handles hosts, game files, collections, configuration, campaigns, maintenance and operation history. -Komodo is not required for native hosts. ## Getting started @@ -134,9 +133,6 @@ affect occupancy or game history. Existing JSON payloads are unchanged. - [Metrics](docs/metrics.md) - [Leaderboard and match history](docs/leaderboard.md) -Older workflow documents may describe the legacy Komodo backend. Native hosts -use the agent boundary described in the native-host document. - ## Development The controller is SvelteKit/TypeScript with SQLite via better-sqlite3, the Node diff --git a/docs/deployment-blueprints.md b/docs/deployment-blueprints.md index 1c4ff57..203a26d 100644 --- a/docs/deployment-blueprints.md +++ b/docs/deployment-blueprints.md @@ -104,7 +104,8 @@ the current shortfall too. All collections and individual launches share one startup queue, ordered by request time. By default, one server starts at a time. Its startup slot remains occupied -until Komodo is healthy, RCON answers, and the reported map matches the selection. +until the host agent confirms container health, game status and the selected map +are verified, and RCON reports a current Plutonium revision. Only then can the next queued server start. Set **Concurrent server startups** in **Settings → Fleet settings** to a positive integer to allow more simultaneous startups. Each worker pass admits at most the available slots, even if an attempt fails immediately. @@ -137,10 +138,10 @@ another launch takes the lobby, submission stops before saving the key; refresh the preview and review the newly assigned ID. Refreshing a preview clears the typed key so it cannot accidentally be saved against a different lobby. -Komodo connection settings, target host, RCON configuration, config filename, -lobby availability, and port availability are checked locally. These checks do -not probe the host or verify game files, images, Docker volumes, firewall -rules, or credentials. Container and RCON verification happen after provisioning. +Host selection, game-file versions, runtime configuration and lobby availability +are checked before queueing. Provisioning assigns host capacity and physical +ports when dispatching. The host agent performs container and RCON verification +locally after launch. See [native hosts](native-hosts.md) for readiness checks. T6 variants retain their exact catalog slug and launch ruleset; RCON health checks verify the engine map, not the variant's start location. diff --git a/docs/maintenance.md b/docs/maintenance.md index fdc4e82..fd5b8d2 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -122,7 +122,7 @@ an application restart, routine watchdog checks wait for a new status-monitor pa Provisioning retains its checks before executing recovery actions. Three observations over at least two minutes are required before an automatic -health restart; starting or restarting containers get five minutes. Unknown Komodo responses are observations, +health restart; starting or restarting containers get five minutes. Unknown host-agent responses are observations, not proof of a stopped container. Repeated manual checks do not accelerate this window. A successful routine or manual game-status probe between scans resets the failure streak. Missing RCON configuration, unresolved ports, and invalid diff --git a/docs/native-hosts.md b/docs/native-hosts.md index ac1d05c..567fded 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -1,8 +1,7 @@ # Native host management -Electric Cherry controls Docker/Plutainer through an outbound host agent. A native -instance never uses Komodo or a filesystem mounted into the controller. The -controller owns desired state, campaigns, collections, safety queues and history; +Electric Cherry controls Docker/Plutainer through an outbound host agent. The controller +does not need mounted game-server files. It owns desired state, campaigns, collections, safety queues and history; the agent owns local execution and its durable operation journal. Fleet health checks query RCON even when container inspection is unavailable. A @@ -554,16 +553,12 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run: execution and after execution but before acknowledgement. Verify the same operation IDs and absence of duplicate containers/restarts. -Legacy backend code remains for existing database registrations. New native -launches do not require its configuration, API, network or mounted directories. - ### Provisioning progress and diagnostics Native launches distinguish initial eligibility evaluation from an actual wait for startup capacity. The operation page records payload preparation, command submission, agent acknowledgement and execution, and separate container, game, -map and Plutonium revision checks. Native actions use host-agent steps rather -than legacy Komodo deployment steps. Evidence remains available after completion. +map and Plutonium revision checks. Evidence remains available after completion. The **Host agent execution** panel reads the durable command state directly, including its queue time and latest report time. Agent receipts are recorded in diff --git a/docs/provisioning-improvement-plan.md b/docs/provisioning-improvement-plan.md index ca8516b..b7657e9 100644 --- a/docs/provisioning-improvement-plan.md +++ b/docs/provisioning-improvement-plan.md @@ -111,7 +111,7 @@ Separate durable milestones from transient readiness checks. A passed container/ | --- | --- | | `canCancelRemoval` | Offer **Cancel queued removal**, explaining that it is available only before removal begins. Preserve the existing atomic worker/cancellation race protection. | | `canResumeVerification` | Offer **Resume health verification** and explain that it observes the existing instance without creating or restarting it. | -| `canResumeCleanup` | Offer **Resume cleanup**, including eligible deleted identities. Explain that Komodo deletion already succeeded. | +| `canResumeCleanup` | Offer **Resume cleanup**, including eligible deleted identities. Explain that instance removal already succeeded. | | `canResumeRebuild` | Offer **Resume rebuild cleanup** and explain that local cleanup precedes creation in the reserved slot. | | Failed/interrupted without a resume capability | Lead with the recorded failure and relevant inspection destination. An uncertain remote response must not produce a generic retry command. | | Cancelled | State that the request is cancelled; distinguish recorded prior work from remaining work that did not run. Do not claim continued execution. | @@ -154,7 +154,7 @@ A suitable small schema extension is nullable `phase_key`, `phase_started_at`, ` Implementation rules: -1. Record structured blockers and observations where the worker already makes decisions. Do not build a second eligibility engine in the API or browser. The detail GET remains read-only and performs no Komodo/RCON probes. +1. Record structured blockers and observations where the worker already makes decisions. Do not build a second eligibility engine in the API or browser. The detail GET remains read-only and performs no host-agent/RCON probes. 2. Derive current capacity from existing scheduler helpers/tables, with an explicit snapshot time. Persist important milestones against the operation because server-owned recovery rows can later be replaced or deleted. Only attach a live slot when its `operation_id` matches; slot absence alone is not evidence of successful release. 3. Update related state, observation, and semantic event atomically around each durable checkpoint, keeping network calls outside database transactions. Dual-write existing stage/event text, especially `REMOVING_SERVER`, which participates in recovery eligibility today. 4. Record meaningful phase, blocker, command, and recovery transitions. Update observation timestamps on repeated checks without appending an event every poll. Changing player counts must not reset the phase's start time. diff --git a/docs/provisioning.md b/docs/provisioning.md index d59d55c..c8626a4 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -12,9 +12,9 @@ server, choose **Manage server** or open its **Server actions** menu. | Create | Create a new instance using the latest promoted files and verify container health, game status, the selected map, and the Plutonium revision. | | Start | Start the existing container and verify container health and RCON. | | Restart | Wait until empty, restart the existing container, and verify container health and RCON. | -| Rebuild | Wait until empty, destroy the instance, delete all server files and its registration, then create a fresh instance in the same server slot and port. Verify container health, game status, map, and Plutonium revision. | -| Stop | Wait until empty, stop the container, and verify that Komodo reports it stopped. Keep the instance for a later Start. | -| Destroy | Wait until empty, remove the container and Komodo deployment, then delete server files and registration. | +| Rebuild | Wait until empty, replace the instance through the host agent, preserving its server identity and port while advancing its generation. Verify container health, game status, map, and Plutonium revision. | +| Stop | Wait until empty, stop the container, and verify that the host agent reports it stopped. Keep the instance for a later Start. | +| Destroy | Wait until empty, remove the container and its files through the host agent, then retire the instance and registration. | Servers linked to a running or ready-to-promote beta cannot be destroyed or rebuilt. Promote or roll back the beta first, including paused or failed test @@ -32,11 +32,10 @@ activity belong to Electric Cherry, and are not instance files. Rebuild reserves the original ID and port throughout teardown and creation, even when a lower lobby is available. It shares the existing creation queue and startup -limit. A failed removal never proceeds to creation. Once Komodo deletion is -confirmed, failed local cleanup offers **Resume rebuild cleanup**; creation cannot -proceed until the old files are fully removed. An interrupted fresh startup can -resume health verification, without repeating teardown or creation. Inspect other -failures before making another request. +limit. The agent records each operation durably, and the controller reconciles its +receipt after reconnecting or restarting. An interrupted fresh startup can resume +health verification without repeating the operation. Inspect failures before +making another request. Create and Rebuild finish only after RCON reports a runtime revision at least as new as the current Plutonium production release. The worker checks the production @@ -49,16 +48,15 @@ updater configuration and server logs before retrying. Telemetry can verify game status and map, but cannot verify the Plutonium revision. Start and Restart share the rolling restart limit with campaigns, betas, and -maintenance. A successful API reply is followed by a recovery delay and actual -container and game-status checks. Failed or ambiguous restart requests retain their recovery +maintenance. A successful agent receipt is followed by actual container and game-status checks. Failed or ambiguous restart requests retain their recovery slot and are not automatically replayed. Stop requires an explicit stopped container state; an -unreachable Komodo API is not evidence that a stop succeeded. +unreachable host agent is not evidence that a stop succeeded. Disruptive operations require a fresh zero-player result from RCON or accepted -telemetry, unless Komodo positively reports a stopped container. Fresh occupied +telemetry, unless the host agent positively reports a stopped container. Fresh occupied telemetry vetoes a conflicting empty RCON reading. Reports must be received within 90 seconds; recovery verification also excludes reports from before the submitted -operation. Telemetry can verify the game's activity and selected map after Komodo +operation. Telemetry can verify the game's activity and selected map after the host agent reports healthy. Runtime actions wait for conflicting campaign, beta, maintenance, or provisioning work. Duplicate pending requests for the same action return the existing operation; a different action returns a @@ -78,8 +76,7 @@ after five minutes; unrelated recovery slots remain held. Drift campaigns follow their linked rebuild without requesting another restart. Their file evidence and history remain available after the instance is removed. -Campaign changes queued before submission can share one restart. Changes arriving -after submission require a new operation. A failed or interrupted request keeps +Each campaign activation has its own durable operation and file selection. A failed or interrupted request keeps its original ID and never automatically replays. An explicit beta retry starts a new attempt, preserving prior history; paused or closed beta work cannot start a queued restart. Recovery of an already submitted restart continues even if its @@ -177,7 +174,7 @@ heartbeat or completion-time promises. literal-text `query`, `limit` (default 25, maximum 100), an exclusive event-ID `cursor`, and an upper-ID `watermark`. Keep the returned watermark when loading more and use `nextCursor` until it is null. Begin without a watermark to include - newly arrived events. These reads do not probe Komodo or RCON. + newly arrived events. These reads do not contact host agents or probe RCON. Old `/deployments` pages redirect to their `/provisioning` equivalents, preserving query parameters. Collection launch links at `/provisioning/collections/:id` or @@ -185,7 +182,6 @@ query parameters. Collection launch links at `/provisioning/collections/:id` or query parameters. `/api/deployments` remains compatible, including old campaign detail endpoints. No history or IDs are renamed or discarded. -Start and Stop use Komodo's corresponding runtime commands. Rebuild uses full -destruction and creation, not Komodo's redeploy command. See the -[Komodo execute API](https://docs.rs/komodo_client/latest/komodo_client/api/execute/index.html) -for the underlying container operations. +Start and Stop use durable `instance.start` and `instance.stop` host-agent +operations. Rebuild uses `instance.rebuild` and replaces the instance files. +See [native host management](native-hosts.md) for command delivery and recovery. diff --git a/docs/t5-t6-idle-cpu-research.md b/docs/t5-t6-idle-cpu-research.md index 8869082..9d3aeb1 100644 --- a/docs/t5-t6-idle-cpu-research.md +++ b/docs/t5-t6-idle-cpu-research.md @@ -1,6 +1,6 @@ # Reducing idle CPU use in T5 and T6 Zombies servers -Research date: September 19, 2026. Scope: Plutonium T5 Zombies (`t5sp`) and T6 Zombies (`t6zm`), with particular attention to Electric Cherry's Plutainer/Docker/Komodo deployment. +Research date: September 19, 2026. Scope: Plutonium T5 Zombies (`t5sp`) and T6 Zombies (`t6zm`), with particular attention to Electric Cherry's Plutainer/Docker host deployment. **Recommendation:** first benchmark Wine's NTSYNC support and audit the effective runtime configuration. If every lobby must remain immediately available, trial CPU limits that apply only while a server is confirmed empty. If spare lobbies can be unavailable temporarily, keep a smaller pool running and start additional servers when demand rises. Stopping unused instances offers the most predictable reduction in their CPU consumption. @@ -60,7 +60,7 @@ The policy should handle several cases explicitly: - Give startup, map changes, and recovery the normal CPU budget. - Restore that budget on a connection attempt if a reliable signal exists. Otherwise, use a conservative idle limit that still supports joining and a detection interval demonstrated to be fast enough. - Restore the active budget when occupancy becomes unknown. A separate watchdog or expiring idle-policy lease should restore it if the controller stops running. -- Coordinate quota ownership with Komodo so container recreation does not silently lose the policy or reinstate an obsolete limit. +- Coordinate quota ownership with the host agent so container recreation does not silently lose the policy or reinstate an obsolete limit. Electric Cherry's current 30-second status cycle can leave a newly joined player under the idle cap until the next successful poll. An aggressive cap should not depend on that cycle alone. Quotas can also allow brief bursts followed by throttling within their scheduling period, so evaluate join time and responsiveness, not just average CPU. The kernel exposes `nr_throttled` and `throttled_usec` in cgroup v2 `cpu.stat` to help identify this behavior. [Linux CPU controller](https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu). diff --git a/docs/ui-ux-implementation.md b/docs/ui-ux-implementation.md index 98557d3..e2b5be5 100644 --- a/docs/ui-ux-implementation.md +++ b/docs/ui-ux-implementation.md @@ -44,7 +44,7 @@ Activity and notifications have been removed. Copy results appear next to their - Promotion uses `GET /api/betas/:id/promote` to review impact and `POST` with the returned token as `previewToken` to apply it. A changed plan returns HTTP 409 and needs renewed review. - Reconciliation uses `POST /api/drift` with `dryRun: true` and optional `servers`, then executes with the returned `expectedFingerprint`. Omitted servers explicitly mean the entire fleet. A stale review returns HTTP 409; changes detected during application are reported as skipped files. - `POST /api/deployments/:id/cancel` cancels remaining provisioning work, including restarts and health checks. It does not undo changes already made. -- System status exposes no credential values. Its optional connection check verifies Komodo network reachability, not credential validity. Remote game-file volumes remain explicitly unverified. +- System status exposes no credential values. It reports host-agent readiness and verified game-file caches. - Drafts contain nonsecret beta setup. Launch keys are not stored in browser drafts. - Log search/export covers the loaded tail, not an entire remote file or a full diagnostic archive. @@ -56,4 +56,4 @@ Activity and notifications have been removed. Copy results appear next to their - Local browser walkthroughs use an isolated database and synthetic populated API fixtures, with production integration credentials disabled. Fleet, server diagnostics, logs, Settings, Leaderboard, libraries, beta setup/progress, launches, keys, and reconciliation are exercised at desktop and phone widths. - Browser checks include the loading button accessibility tree, clickable native labels, URL/Back restoration, retained previews/drafts, selected-tab visibility, pending import ownership, and horizontal overflow. These are targeted checks, not a claim of complete accessibility conformance. -Live Komodo provisioning, real game-server restarts and removals were not executed during verification. Their boundary behavior is covered by regression tests and simulated UI flows. +Live host provisioning, real game-server restarts and removals were not executed during verification. Their boundary behavior is covered by regression tests and simulated UI flows. diff --git a/docs/ui-ux-review.md b/docs/ui-ux-review.md index 49214d8..b562aa9 100644 --- a/docs/ui-ux-review.md +++ b/docs/ui-ux-review.md @@ -10,7 +10,7 @@ Electric Cherry has a coherent visual foundation and unusually capable operation This review combines source inspection with a local browser walkthrough. It covers Fleet, server details and diagnostics, Scripts, Configs, Plugins, beta creation and progress, server deployments, blueprints, campaigns, keys, and Leaderboard. Existing design notes and workflow documentation informed the review. -Browser exploration used an isolated database with Komodo/RCON credentials disabled. Nine top-level pages were examined at 1440×1000 and 390×844. Populated Fleet, server details, and beta scenarios used synthetic API responses. Beta setup, blueprint validation, and the browser accessibility tree were also exercised. Navigation was checked around its responsive breakpoint. Production integrations, actual restarts, and destructive actions were not executed. +Browser exploration used an isolated database with external connections disabled. Nine top-level pages were examined at 1440×1000 and 390×844. Populated Fleet, server details, and beta scenarios used synthetic API responses. Beta setup, blueprint validation, and the browser accessibility tree were also exercised. Navigation was checked around its responsive breakpoint. Production integrations, actual restarts, and destructive actions were not executed. **Evidence labels:** **Rendered** means reproduced in the local browser, sometimes with an explicitly synthetic populated state. **Source** means verified in UI/API implementation. **Opportunity** means a proposed capability or design direction, rather than a defect. Recommendations are expert judgments; they are not findings from end-user interviews or a complete accessibility conformance audit. @@ -198,7 +198,7 @@ Sources: [Scripts](../src/routes/scripts/+page.svelte), lines 13–23; [Plugins] Settings currently leads only to server keys. Integration information is available deep in server diagnostics; the empty Fleet encourages creation even when external services are not configured. The first-run user needs a short path from prerequisites to a functioning lobby. -Add a read-only System status page and a contextual first-run checklist: Komodo configured/reachable, RCON configured, game files available where verifiable, keys available, and first server ready. Link directly to relevant setup instructions. Distinguish **not configured**, **unreachable**, and **server offline**. Avoid repeatedly displaying the full checklist after setup succeeds. +Add a read-only System status page and a contextual first-run checklist: host agents enrolled/connected, local RCON responding, verified game files available, keys available, and first server ready. Link directly to relevant setup instructions. Distinguish **not configured**, **unreachable**, and **server offline**. Avoid repeatedly displaying the full checklist after setup succeeds. Acceptance: someone with an empty installation can identify the next missing prerequisite without opening an unrelated server or reading the full README. @@ -306,7 +306,7 @@ The visual language is coherent, but small controls, nested cards, repeated sect Use a dedicated editable-control boundary token, reserve strong surfaces for actionable sections, and reduce repeated title/subtitle layers in builders. Keep body text and important form instructions comfortably readable; use compact metadata selectively. Add descriptive route titles, especially server name and lobby number. -Use “server” consistently for the managed unit; introduce “lobby” as its player-facing meaning. Prefer **Preview missing servers** over “Conform fleet.” Use “Files” consistently for scripts/configs/plugins where a collective noun is needed. Keep raw IDs, RCON, Komodo, and hashes accessible in technical details. Expose the existing keyboard shortcuts through a small help panel. +Use “server” consistently for the managed unit; introduce “lobby” as its player-facing meaning. Prefer **Preview missing servers** over “Conform fleet.” Use “Files” consistently for scripts/configs/plugins where a collective noun is needed. Keep raw IDs, RCON, host-agent status, and hashes accessible in technical details. Expose the existing keyboard shortcuts through a small help panel. Acceptance: primary tasks are identifiable by placement and wording, and multiple browser tabs are distinguishable. Contrast should be measured before claiming a conformance result. diff --git a/komodo/stack.toml b/komodo/stack.toml deleted file mode 100644 index 4811b5c..0000000 --- a/komodo/stack.toml +++ /dev/null @@ -1,41 +0,0 @@ -# Komodo stack definition. Deploys electric-cherry as a managed container on -# the same host running the T5/T6 Plutonium servers. Mounts /root/zombies so the -# app can push scripts to the same paths push_scripts.sh used, and dedicates -# a small subdir for its SQLite state. - -[[stack]] -name = "electric-cherry" - -[stack.services.web] -image = "electric-cherry:latest" -restart = "unless-stopped" -ports = ["3000:3000"] - -volumes = [ - "/root/zombies:/root/zombies:rw", - "/root/zombies/.electric-cherry:/data:rw" -] - -# Lets RCON_HOST=host.docker.internal reach game servers running on the -# same Komodo host without knowing each container's internal IP. -extra_hosts = ["host.docker.internal:host-gateway"] - -[stack.services.web.env] -SERVERS_ROOT = "/root/zombies/t5" -T6_SERVERS_ROOT = "/root/zombies/t6" -PLUTAINER_T6_GAMEFILES_VOLUME = "t6_game_files" -REL_DEST = "runtime/plutonium/storage/t5/raw/scripts/sp" -BETA_ARCHIVE_DIR = "/root/zombies/beta-archive" -DB_PATH = "/data/cherry.db" - -# secrets: bind via Komodo's env-file or secret store -KOMODO_ADDRESS = "https://komodo.example.internal" -KOMODO_API_KEY = "" -KOMODO_API_SECRET = "" -# Host new deployments land on — must match the name shown in Komodo's UI. -KOMODO_TARGET_SERVER = "" - -# RCON — shared password across every Plutonium server in the fleet. -RCON_PASSWORD = "" -RCON_HOST = "host.docker.internal" -RCON_TIMEOUT_MS = "3000" diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index 2e9c919..5742c31 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -1,12 +1,12 @@ import type { ProvisioningAction } from './provisioning.js'; import type { LaunchRecipeSummary } from './deployment-plan.js'; -export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', 'create-deployment', - 'container-action', 'remove-container', 'delete-deployment', 'cleanup', 'verify-container', 'verify-game', +export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', + 'container-action', 'cleanup', 'verify-container', 'verify-game', 'verify-map', 'verify-plutonium', 'verify-stopped'] as const; export type ProvisioningPhase = typeof provisioningPhases[number]; export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown'; -export type EvidenceSource = 'komodo' | 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null; +export type EvidenceSource = 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null; export interface ProvisioningCheck { key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped'; @@ -98,7 +98,7 @@ export interface ProvisioningDetail { canCancel?: boolean; canCancelCreation?: boolean; cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null; - backend?: 'native' | 'komodo'; + backend?: 'native'; workerActivity?: WorkerActivity[]; agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null; deployment: ProvisioningDeployment; diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index 346c8c3..3ceae46 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -33,31 +33,7 @@ export const config = { betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'), dbPath: req('DB_PATH', './data/cherry.db'), httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000), - komodo: { - address: env.KOMODO_ADDRESS ?? '', - apiKey: env.KOMODO_API_KEY ?? '', - apiSecret: env.KOMODO_API_SECRET ?? '', - // Komodo server (host) new deployments land on. Required for the - // deploy flow — Komodo needs to know which machine to schedule the - // container onto. Look up the exact name in Komodo's UI (Servers page). - targetServer: env.KOMODO_TARGET_SERVER ?? '', - // Default image for freshly-provisioned T5 and T6 zombie servers. - // Overridable per-deploy in the UI later if we ever need to. - defaultImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest', - defaultNetwork: env.PLUTAINER_NETWORK ?? 'komodo_default', - // Named docker volume that holds the read-only game files layer. - gameFilesVolume: env.PLUTAINER_GAMEFILES_VOLUME ?? 't5_game_files', - t6GameFilesVolume: env.PLUTAINER_T6_GAMEFILES_VOLUME ?? 't6_game_files' - }, - // Shared across every newly-provisioned server. Kept in sync with the - // fleet's RCON_PASSWORD (rcon.password) so freshly-created servers respond - // to the same rcon secret the sweep already uses. - // Port range Electric Cherry auto-allocates from when creating a new - // deployment. Update this when the host's firewall opens more. - portRange: { - min: num('SERVER_PORT_MIN', 28960), - max: num('SERVER_PORT_MAX', 28980) - }, + defaultRuntimeImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest', rcon: { // Host every RCON packet is sent to. Defaults to the Docker // host-gateway alias so the electric-cherry container can reach @@ -70,10 +46,6 @@ export const config = { } } as const; -export function komodoConfigured(): boolean { - return !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret); -} - export function rconConfigured(): boolean { return !!config.rcon.password; } diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 826cc6f..b0e3f90 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -24,7 +24,6 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ { name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" }, // Base columns that shipped in the very first schema stay here for parity // with fresh installs — no ALTER is issued if they're already present. - { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' }, { name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' }, { name: 'rcon_host', ddl: 'ALTER TABLE servers ADD COLUMN rcon_host TEXT' }, { name: 'last_players', ddl: 'ALTER TABLE servers ADD COLUMN last_players INTEGER' }, @@ -65,6 +64,14 @@ function migrateServers(handle: Database.Database): void { handle.exec(col.ddl); } } + // Retire external-backend registry fields, including required fields that + // would otherwise prevent native inserts into an upgraded database. + const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']); + handle.transaction(() => { + for (const column of rows) { + if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`); + } + })(); } // Additive columns on tables the script model owns. Same pattern as @@ -286,7 +293,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void { function backfillDesiredState(handle: Database.Database): void { if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return; handle.transaction(() => { - // A failed stop may have reached Komodo. Preserve that explicit intent. + // A failed stop may have reached the host agent. Preserve that explicit intent. handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = ( SELECT action FROM server_deployments d WHERE d.server = servers.name AND d.state != 'cancelled' AND d.action IN ('create','start','restart','rebuild','stop') diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index b9017c0..c203068 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -63,13 +63,10 @@ CREATE INDEX IF NOT EXISTS idx_leaderboard_player_matches ON leaderboard_match_p CREATE TABLE IF NOT EXISTS servers ( name TEXT PRIMARY KEY, - komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container - komodo_name TEXT NOT NULL, - komodo_server TEXT, container_checked_at TEXT, - container_state TEXT, -- last observed Komodo runtime state + container_state TEXT, -- last observed host-agent runtime state desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')), - game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo + game_port INTEGER, -- UDP game port assigned on the host rcon_host TEXT, -- optional per-server override of RCON_HOST game TEXT, -- 't5' etc.; static per-server (no map rotation) map TEXT, -- 'zombie_theater' etc.; static per-server diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 9436f12..7a739c0 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -376,7 +376,7 @@ export async function openBeta(input: OpenBetaInput): Promise { const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health') OR (state IN ('failed', 'interrupted') AND stage IN ( - 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'))) + 'Stopping and removing container', 'Removing server files and registry'))) LIMIT 1`).get(name); if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict'); serverRows.push({ name, game: row.game, map: row.map }); diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 6b4ab58..26da11c 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -2,16 +2,14 @@ import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js'; import { db } from '../db/index.js'; -import { config, komodoConfigured, rconConfigured } from '../config.js'; import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js'; -import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js'; +import { creationQueue, destroyServer } from '../provisioning/index.js'; import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js'; import { housePartyTargets } from './house-party.js'; import { serverObservation } from './status.js'; import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js'; -import { listHosts } from '../hosts/index.js'; import { nativeDeployment, nativeInstance } from '../backends/native.js'; import { launchAvailability } from '../hosts/availability.js'; @@ -71,7 +69,7 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, include return { counts, retainedServers, removals }; } -function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] { +function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] { const occupied = db().prepare(`SELECT name FROM servers UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health') @@ -85,16 +83,14 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co taken.add(Number(match[2])); lobbies.set(match[1], taken); } const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id)); - const ports: number[] = []; return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => { const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; const taken = lobbies.get(base) ?? new Set(); const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null; const name = lobby === null ? null : `${base}_${lobby}`; const native = entry.recipe.native; - const port = native ? null : nextAvailablePort(ports, releasingServers); + const port = null; if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); } - if (port !== null) ports.push(port); return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) }; })); } @@ -102,13 +98,8 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] { const blockers: string[] = []; if (!servers.length) return blockers; - const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native); - if (legacy.length) { - if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.'); - if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.'); - } + if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.'); if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); - if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`); return blockers; } @@ -120,7 +111,7 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] { AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined; return !!creation && nativeDeployment(creation.id); }; - if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.'); + if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.'); for (const server of removals) { const blocker = activeBetaRemovalBlocker(server.name); if (blocker) blockers.push(blocker); @@ -133,15 +124,13 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); return db().transaction((): CollectionPreview => { const { entries, blueprintName } = blueprintEntries(blueprintId); - if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) { - const libraries = selection?.libraryIds as Record | undefined; - for (const entry of entries) { - entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); - entry.recipe.image = entry.recipe.native.image!; - } + const libraries = selection?.libraryIds as Record | undefined; + for (const entry of entries) { + entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); + entry.recipe.image = entry.recipe.native.image!; } const { counts, retainedServers, removals } = collectionPlan(entries, mode); - const servers = allocate(entries, removals.map(server => server.name), counts); + const servers = allocate(entries, counts); const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; const selections = servers.flatMap(server => server.native ? [server.native] : []); const availability = selections.length ? launchAvailability(selections) : {}; @@ -176,16 +165,12 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'); } const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false); - const releasingServers = removals.map(server => server.name); - const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native); - // Native Apply accepts the reviewed logical identities. Capacity and - // physical ports can change while this request waits in the queue. - const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts); + const servers = reviewed; const validatedSelections = new Set(); for (const server of servers) { const recipe = entries[server.entryIndex].recipe; const selection = server.native ?? recipe.native; - if (!selection) continue; + if (!selection) throw new Error('Select a native host before launching. Refresh the preview.'); const identity = JSON.stringify([recipe.game, selection]); if (validatedSelections.has(identity)) continue; // Recheck saved configuration once per selection without allocating @@ -195,9 +180,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): } const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); - if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length) - || servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) { - throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.'); + if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) { + throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.'); } const expected = new Set(servers.map(server => server.name)); for (const [name, value] of Object.entries(keys)) { @@ -207,10 +191,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const collectionId = randomUUID(); db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)') .run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers)); - const removalJobs = removals.map((server, position) => { - const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION - SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port); + removals.forEach((server, position) => { const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined; let id: string; if (queued && !getServer(server.name)) { @@ -230,27 +211,17 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): id = result.deploymentId; } db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position); - return { id, ports }; }); servers.forEach((server, index) => { const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) }; - let result; - if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug, + if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.'); + const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug, selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name }); - else { - const childPreviewId = randomUUID(); - db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)') - .run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name); - result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug, - expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers); - } // Throw so the outer transaction also reverses all earlier keys and children. if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`); db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)') .run(collectionId, result.deploymentId, index); - for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) { - db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id); - } + }); db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id); return { ok: true as const, collectionId }; diff --git a/src/lib/server/fleet/health-observations.ts b/src/lib/server/fleet/health-observations.ts index d641738..de1d5b6 100644 --- a/src/lib/server/fleet/health-observations.ts +++ b/src/lib/server/fleet/health-observations.ts @@ -3,7 +3,7 @@ import { config } from '../config.js'; import { db } from '../db/index.js'; import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js'; import { nativeInstance, type NativeInstance } from '../backends/native.js'; -import { komodoInspectHealthy } from '../integrations/komodo.js'; +import { nativeHealth } from '../backends/native.js'; import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js'; import { withOperationSource } from '../hosts/operation-context.js'; import { getServer } from './servers.js'; @@ -39,7 +39,6 @@ function identity(server: string, probe: Probe): Identity { return { key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle, native ? [native.host_id, native.id, native.generation] : null, - native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs], probe === 'rcon' ? [row.rcon_host, config.rcon] : null]), port: probe === 'rcon' ? native?.port ?? row.game_port : null, checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at @@ -77,13 +76,7 @@ async function observe(server: string, fresh: boolean while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!); const pending = Promise.resolve().then(() => request(server)).then(result => { const latest = identity(server, probe); - // Legacy RCON may discover and save its first port during this request. - // Accept that enrichment only when all other identity fields agree. - const endpoint = 'endpoint' in result ? result.endpoint : null; - const learnedPort = probe === 'rcon' && !current.port && result.ok - && endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port; - if (current.key !== latest.key || (current.port !== latest.port && !learnedPort) - || newerEvidence(latest, entry.startedAt)) { + if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) { throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.'); } const observation = { result, observedAt: new Date().toISOString() }; @@ -100,12 +93,12 @@ async function observe(server: string, fresh: boolean return pending; } -const inspections = new Map>>>(); +const inspections = new Map>>>(); const rcon = new Map>>>(); interface MonitoringStatus { identity: string; - container: Observation>>; + container: Observation>>; game?: Observation>>; } // Keep failed samples too, independently of the successful-probe cache. Reading @@ -160,7 +153,7 @@ export function monitoringStatus(server: string): MonitoringStatus | null { } /** Keep the actual sample time: a later heartbeat must not renew old evidence. */ -export function heartbeatInspection(server: string): Observation>> | null { +export function heartbeatInspection(server: string): Observation>> | null { const row = getServer(server); if (!row?.native_instance_id || !row.host_id) return null; const host = getHost(row.host_id); @@ -242,7 +235,7 @@ export function inspectForMonitoring(server: string, fresh = false) { if (fresh) statuses.delete(server); const heartbeat = !fresh && heartbeatInspection(server); if (heartbeat) return Promise.resolve(heartbeat); - return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy); + return observe(server, fresh, 'inspect', inspections, nativeHealth); } export function rconForMonitoring(server: string, fresh = false) { if (fresh) statuses.delete(server); diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 32eb5e2..34a797c 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -105,7 +105,7 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean) } addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!); return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical, - capturedAt: new Date().toISOString(), image: config.komodo.defaultImage, + capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage, configFile: configFileForGame(game), configSource, files }; } diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index 05101a7..729e693 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -74,9 +74,6 @@ export interface ServerRow { instance_generation?: number | null; game_library_id?: string | null; name: string; - komodo_type: 'deployment' | 'stack' | 'container'; - komodo_name: string; - komodo_server: string | null; container_state: string | null; container_checked_at: string | null; desired_state: 'running' | 'stopped'; @@ -112,12 +109,10 @@ export interface ServerRow { export async function seedServerRegistry(): Promise { // Explicit legacy import only; never run as part of application startup. const rows = await serverDirectories(); - // Under the new naming scheme Komodo entities match EC server names 1:1 - // (both underscore-separated), so komodo_name = name — no transformation. const insert = db().prepare( `INSERT OR IGNORE INTO servers - (name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir) - VALUES (?, 'deployment', ?, ?, ?, ?, ?, ?)` + (name, game, map, game_mode, map_slug, host_dir) + VALUES (?, ?, ?, ?, ?, ?)` ); // Backfill existing rows where derived columns were never set. game_mode // and map_slug are new; older rows can still surface them from the name. @@ -139,7 +134,7 @@ export async function seedServerRegistry(): Promise { const gameMode = parsed?.gameMode ?? null; const mapSlug = parsed?.mapSlug ?? null; const hostDir = serverDir(name); - const res = insert.run(name, name, game, map, gameMode, mapSlug, hostDir); + const res = insert.run(name, game, map, gameMode, mapSlug, hostDir); if (res.changes > 0) { n++; } else { @@ -185,9 +180,6 @@ export function setServerField( export function setServerFields(name: string, fields: Partial>): void { const allowed: Array = [ - 'komodo_type', - 'komodo_name', - 'komodo_server', 'container_state', 'container_checked_at', 'game_port', diff --git a/src/lib/server/fleet/snapshot.ts b/src/lib/server/fleet/snapshot.ts index 9c63e76..c21ec64 100644 --- a/src/lib/server/fleet/snapshot.ts +++ b/src/lib/server/fleet/snapshot.ts @@ -1,7 +1,7 @@ import { db } from '../db/index.js'; import { getServer, listServers } from './servers.js'; import { observationReader, serverObservation } from './status.js'; -import { config, komodoConfigured, rconConfigured } from '../config.js'; +import { config, rconConfigured } from '../config.js'; import { runningVersions } from '../integrations/plutonium.js'; import { listHosts } from '../hosts/index.js'; import type { NativeInstance } from '../backends/native.js'; @@ -33,8 +33,7 @@ export function fleetSnapshot(serverName?: string) { host: s.host_id ? hostsById.get(s.host_id) ?? null : null, network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null, plutonium: versions.get(s.name), - // Derived endpoint — same logic as endpointFor() but without asking - // Komodo. If we don't have a port yet, endpoint is null. + // The host agent owns the game connection. endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, held: holds.get(s.name) ?? [] }; @@ -44,11 +43,10 @@ export function fleetSnapshot(serverName?: string) { serverNow: Date.now(), reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [], integrations: { - native: hosts.length > 0 || !komodoConfigured(), + native: true, hostsReady: hosts.filter(host => host.readiness === 'ready').length, hostsTotal: hosts.length, rcon: rconConfigured(), - komodo: komodoConfigured(), rconTimeoutMs: config.rcon.timeoutMs, defaultRconHost: rconHost } diff --git a/src/lib/server/fleet/status-evidence.ts b/src/lib/server/fleet/status-evidence.ts index 4721c37..380dd03 100644 --- a/src/lib/server/fleet/status-evidence.ts +++ b/src/lib/server/fleet/status-evidence.ts @@ -59,7 +59,7 @@ export async function probePlayerEvidence(server: string): Promise { diff --git a/src/lib/server/fleet/status.ts b/src/lib/server/fleet/status.ts index b5cbf69..c492e94 100644 --- a/src/lib/server/fleet/status.ts +++ b/src/lib/server/fleet/status.ts @@ -7,7 +7,6 @@ import { telemetryReport, latestGameSubmission } from './status-evidence.js'; import { latestGameSubmissions } from './lifecycle-evidence.js'; import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; -import { restartRequestInFlight } from '../provisioning/restarts.js'; interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null } @@ -33,7 +32,7 @@ export function observationReader() { const operation = operations.get(server.name), restart = restarts.get(server.name); return serverObservation(server, { provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state), - settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000), + settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000), telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null }); }; diff --git a/src/lib/server/hosts/capacity-overview.ts b/src/lib/server/hosts/capacity-overview.ts index 7628a91..355cdbf 100644 --- a/src/lib/server/hosts/capacity-overview.ts +++ b/src/lib/server/hosts/capacity-overview.ts @@ -32,7 +32,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise games.includes(game))) reasons.push('Latest game files for this selection are not ready.'); if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.'); const eligible = connected && host.readiness === 'ready' && reasons.length === 0; diff --git a/src/lib/server/hosts/preparation.ts b/src/lib/server/hosts/preparation.ts index 5ea236b..e96c8bb 100644 --- a/src/lib/server/hosts/preparation.ts +++ b/src/lib/server/hosts/preparation.ts @@ -88,7 +88,7 @@ function targets(host: Host): Target[] { && operation.sequence > enabling.operation_floor && operation.state === 'completed' && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at); const selections = queuedSelections(host.id); - const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]); + const images = new Set([config.defaultRuntimeImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]); const result: Target[] = [...images].map(image => { const operation = latestOperation(host.id, 'runtime.pull', 'image', image); const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts deleted file mode 100644 index c7bd554..0000000 --- a/src/lib/server/integrations/komodo.ts +++ /dev/null @@ -1,832 +0,0 @@ -import { config, komodoConfigured } from '../config.js'; -import { getServer } from '../fleet/servers.js'; -import { nativeHealth, nativeInstance } from '../backends/native.js'; - -// Ports `komodo_restart` in push_scripts.sh:622. Three shapes depending on -// the tracked komodo_type: deployment | container | stack. - -export type KomodoRestartResult = - | { ok: true } - | { ok: false; kind: 'unconfigured' | 'no-name' | 'http-error' | 'unreachable'; message: string }; - -async function komodoCall(path: string, body: unknown): Promise { - const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), config.httpTimeoutMs); - try { - return await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Api-Key': config.komodo.apiKey, - 'X-Api-Secret': config.komodo.apiSecret - }, - body: JSON.stringify(body), - signal: controller.signal - }); - } finally { - clearTimeout(timer); - } -} - -export async function komodoRestart(serverName: string): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - const row = getServer(serverName); - if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' }; - - const type = row.komodo_type ?? 'deployment'; - let path: string; - let body: Record; - - switch (type) { - case 'stack': - path = 'execute/RestartStack'; - body = { stack: row.komodo_name }; - break; - case 'container': - if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' }; - path = 'execute/RestartContainer'; - body = { server: row.komodo_server, container: row.komodo_name }; - break; - case 'deployment': - default: - path = 'execute/RestartDeployment'; - body = { deployment: row.komodo_name }; - break; - } - - const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; - - let res: Response; - try { - res = await komodoCall(path, body); - } catch (err) { - return { ok: false, kind: 'unreachable', message: describeFetchError(err, url) }; - } - if (!res.ok) { - return { ok: false, kind: 'http-error', message: `HTTP ${res.status}` }; - } - - // The script also checks the response body: if it's an object with - // .success === false, treat as failure. If .success is absent, trust the - // HTTP status (line 646). - try { - const json = await res.json(); - if (json && typeof json === 'object' && 'success' in json && json.success === false) { - return { ok: false, kind: 'http-error', message: 'komodo returned success=false' }; - } - } catch { - // Non-JSON body is fine — HTTP 2xx already indicates success. - } - return { ok: true }; -} - -// ---------- Port resolution ---------- - -export type KomodoPortFailure = - | 'unconfigured' - | 'no-name' - | 'unreachable' // network / DNS / timeout - | 'http-error' // non-2xx - | 'unparseable' // 2xx but body wasn't JSON - | 'no-udp-in-response'; // parsed but no UDP port found - -export interface KomodoPortSuccess { - ok: true; - port: number; - snippet: string; - url: string; -} -export interface KomodoPortFailureResult { - ok: false; - reason: KomodoPortFailure; - detail: string; - httpStatus?: number; - snippet?: string; - // URL we tried (or would have tried) — surfaced in the UI so operators - // can `curl` it from the container to verify reachability. - url?: string; -} -export type KomodoPortResult = KomodoPortSuccess | KomodoPortFailureResult; - -const SNIPPET_LIMIT = 4000; - -function truncate(s: string): string { - if (s.length <= SNIPPET_LIMIT) return s; - return s.slice(0, SNIPPET_LIMIT) + `\n… (${s.length - SNIPPET_LIMIT} more bytes)`; -} - -/** - * Turn a Node/Undici fetch failure into something an operator can act on. - * The default `.message` for an aborted fetch is just "This operation was - * aborted" — useless. For network failures Node hides the real cause under - * `.cause` (e.g. `ECONNREFUSED`, `ENOTFOUND`, `EHOSTUNREACH`). Unwrap both - * and always include the URL we tried. - */ -function describeFetchError(err: unknown, url: string): string { - const e = err as (Error & { name?: string; cause?: unknown }) | undefined; - if (e?.name === 'AbortError') { - return `timed out after ${config.httpTimeoutMs}ms dialing ${url}`; - } - const cause = e?.cause as (Error & { code?: string }) | undefined; - if (cause && (cause.code || cause.message)) { - const code = cause.code ? `${cause.code}: ` : ''; - return `${code}${cause.message ?? 'network error'} (dialing ${url})`; - } - return `${e?.message ?? String(err)} (dialing ${url})`; -} - -function safeJsonStringify(value: unknown): string { - try { - return JSON.stringify(value, null, 2); - } catch { - return String(value); - } -} - -/** - * Detailed variant of `komodoResolveGamePort` — returns a discriminated - * union so callers can distinguish "network died" from "auth failed" from - * "we parsed the JSON but couldn't find a UDP port in a shape we recognize". - * - * On success, the response snippet is included so the caller can render it - * as "here's where we found the port". On no-udp-in-response, the snippet - * is the only way we'll know what shape Komodo actually emitted — surface - * it prominently in the UI so we can extend `extractUdpPort` for that shape. - */ -export async function komodoResolvePortDetailed(serverName: string): Promise { - if (!komodoConfigured()) { - return { ok: false, reason: 'unconfigured', detail: 'KOMODO_ADDRESS/KEY/SECRET are not all set' }; - } - const row = getServer(serverName); - if (!row?.komodo_name) { - return { ok: false, reason: 'no-name', detail: `server has no komodo_name: ${serverName}` }; - } - - const type = row.komodo_type ?? 'deployment'; - let path: string; - let body: Record; - - switch (type) { - case 'stack': - path = 'read/GetStack'; - body = { stack: row.komodo_name }; - break; - case 'container': - if (!row.komodo_server) { - return { ok: false, reason: 'no-name', detail: 'container type needs komodo_server' }; - } - path = 'read/InspectContainer'; - body = { server: row.komodo_server, container: row.komodo_name }; - break; - case 'deployment': - default: - path = 'read/GetDeployment'; - body = { deployment: row.komodo_name }; - break; - } - - const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; - - let res: Response; - try { - res = await komodoCall(path, body); - } catch (err) { - return { ok: false, reason: 'unreachable', detail: describeFetchError(err, url), url }; - } - - // Read the body up front — we want the snippet even on non-2xx so the - // UI can show what the error page actually said. - const rawText = await res.text().catch(() => ''); - - if (!res.ok) { - return { - ok: false, - reason: 'http-error', - detail: `HTTP ${res.status} ${res.statusText || ''}`.trim(), - httpStatus: res.status, - snippet: truncate(rawText), - url - }; - } - - let json: unknown; - try { - json = JSON.parse(rawText); - } catch (err) { - return { - ok: false, - reason: 'unparseable', - detail: `response body is not JSON: ${(err as Error).message}`, - httpStatus: res.status, - snippet: truncate(rawText), - url - }; - } - - const port = extractUdpPort(json); - const snippet = truncate(safeJsonStringify(json)); - if (port === null) { - return { - ok: false, - reason: 'no-udp-in-response', - detail: 'parsed the response but no UDP port matched any known shape', - httpStatus: res.status, - snippet, - url - }; - } - return { ok: true, port, snippet, url }; -} - -/** - * Query Komodo's read API for a server's UDP game port. Called lazily by - * the RCON client and cached into `servers.game_port` on success. Thin shim - * over `komodoResolvePortDetailed` that swallows diagnostics — use the - * detailed variant when you care why it failed. - */ -export async function komodoResolveGamePort(serverName: string): Promise { - const result = await komodoResolvePortDetailed(serverName); - return result.ok ? result.port : null; -} - -// ---------- Health probe ---------- - -export type KomodoHealthResult = - | { ok: true; state: string } - | { ok: false; reason: string; state?: string; snippet?: string }; - -// Komodo's container summary has { state: "running", container: { status: -// "Up 35 seconds (health: starting)" } }. Inspect responses instead have -// State.Health.Status (or snake/lowercase equivalents). Runtime health wins -// over a coarse running flag. Unknown/configuration strings are not readiness. -export function extractContainerState(payload: unknown): string | null { - const seen = new Set(); - const known = new Set(['running','healthy','starting','restarting','unhealthy','stopped','exited','created','dead','paused','removing','notdeployed','unknown']); - const stateOf = (value: unknown): string | null => { - if (typeof value !== 'string') return null; - const state = value.toLowerCase().trim(); - if (known.has(state)) return state; - if (/^up\b/.test(state)) { - const health = state.match(/\((?:health:\s*)?(starting|unhealthy|healthy)\)/)?.[1]; - return health && health !== 'healthy' ? health : 'running'; - } - if (/^exited\b/.test(state)) return 'exited'; - if (/^restarting\b/.test(state)) return 'restarting'; - return null; - }; - const walk = (value: unknown): string | null => { - if (!value || typeof value !== 'object' || seen.has(value)) return null; - seen.add(value); - if (Array.isArray(value)) { - const states = value.map(walk).filter((state): state is string => !!state); - return states.find(state => !['running','healthy'].includes(state)) ?? states[0] ?? null; - } - const object = value as Record; - const rawState = object.State ?? object.state; - const state = stateOf(rawState) ?? stateOf(object.container_state ?? object.containerState ?? object.deployment_state ?? object.deploymentState); - // A stopped process is never ready just because its previous health check passed. - if (state && !['running','healthy','unknown'].includes(state)) return state; - if (rawState && typeof rawState === 'object') { - const nested = walk(rawState); - if (nested) return nested; - } - for (const key of ['container','Container','containers','Containers']) { - if (object[key] === null && state === 'running') return 'unknown'; - const nested = walk(object[key]); - if (nested) return nested; - } - const status = stateOf(object.Status ?? object.status ?? object.docker_status); - if (status && !['running','healthy'].includes(status)) return status; - const health = object.Health ?? object.health; - const healthState = stateOf(typeof health === 'object' && health ? - (health as Record).Status ?? (health as Record).status : health); - if (healthState && healthState !== 'healthy') return healthState; - if (state || status) return state ?? status; - if (object.Running === true || object.running === true) return healthState ?? 'running'; - for (const [key, item] of Object.entries(object)) { - if (['config','Config','labels','Labels'].includes(key)) continue; - const nested = walk(item); - if (nested) return nested; - } - return null; - }; - return walk(payload); -} - -const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']); - -async function fetchJsonWithSnippet( - path: string, - body: Record -): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> { - const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; - let res: Response; - try { - res = await komodoCall(path, body); - } catch (err) { - return { ok: false, reason: describeFetchError(err, url) }; - } - const text = await res.text().catch(() => ''); - if (!res.ok) { - const unsupportedEndpoint = res.status === 404 || ( - [400, 405, 422, 500, 501].includes(res.status) - && /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text) - ); - return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint }; - } - let json: unknown; - try { - json = JSON.parse(text); - } catch (err) { - return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) }; - } - return { ok: true, json, snippet: truncate(safeJsonStringify(json)) }; -} - -/** - * Ask Komodo whether the server's container is currently running. Used by the - * sweep during the health phase (post-restart) and by the beta runner. For - * deployment-type servers we hit the container-inspect endpoint first because - * GetDeployment returns config, not runtime state — falling back only when - * the container endpoint isn't available. - */ -export async function komodoInspectHealthy(serverName: string): Promise { - if (getServer(serverName)?.native_instance_id) return nativeHealth(serverName); - if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' }; - const row = getServer(serverName); - if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' }; - - const type = row.komodo_type ?? 'deployment'; - const attempts: Array<{ path: string; body: Record }> = []; - switch (type) { - case 'stack': - attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } }); - break; - case 'container': - if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' }; - attempts.push({ - path: 'read/InspectContainer', - body: { server: row.komodo_server, container: row.komodo_name } - }); - break; - case 'deployment': - default: - // Container-specific endpoints first — they return docker-inspect - // shape with State.Status. GetDeployment is the last-resort - // config-only fallback. - attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } }); - attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } }); - attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } }); - attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } }); - break; - } - - let lastSnippet: string | undefined; - for (const { path, body } of attempts) { - const r = await fetchJsonWithSnippet(path, body); - if (!r.ok) { - // Alternate endpoints handle API-version differences. Retrying an - // outage against every endpoint delays all other server probes. - if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet }; - lastSnippet = r.snippet ?? lastSnippet; - continue; - } - const state = extractContainerState(r.json); - if (state) { - const healthy = HEALTHY_STATES.has(state.toLowerCase()); - if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet }; - return { ok: true, state }; - } - lastSnippet = r.snippet; - } - return { - ok: false, - reason: 'no container state found in any known Komodo response shape', - snippet: lastSnippet - }; -} - -// ---------- Write / Execute (create + destroy deployments) ---------- - -export type KomodoWriteResult = - | { ok: true } - | { ok: false; kind: 'unconfigured' | 'http-error' | 'unreachable'; message: string; url?: string; httpStatus?: number }; - -/** - * Env var pairs get flattened into a single newline-delimited string, matching - * Komodo's DeploymentConfig.environment shape. - */ -export interface DeploymentEnvVar { - variable: string; - value: string; -} - -export interface CreateDeploymentSpec { - name: string; // Komodo deployment name (== EC server name) - server: string; // Komodo server (host) id or name - image: string; // 'ghcr.io/ayymoss/plutainer:latest' - network?: string; // 'komodo_default' - ports: string[]; // ['28960:28960/udp', '28960:28960/tcp'] - volumes: string[]; // ['/host/path:/container/path', 'namedvol:/container/path:ro'] - environment: DeploymentEnvVar[]; -} - -async function postAndClassify(path: string, body: unknown): Promise { - const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; - let res: Response; - try { - res = await komodoCall(path, body); - } catch (err) { - return { ok: false, kind: 'unreachable', message: describeFetchError(err, url), url }; - } - if (!res.ok) { - const snippet = await res.text().catch(() => ''); - return { - ok: false, - kind: 'http-error', - httpStatus: res.status, - message: `HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`, - url - }; - } - // Some Komodo endpoints return `{ success: false }` with a 200. Treat that - // as a failure so callers can surface the real problem. - try { - const json = await res.json(); - if (json && typeof json === 'object' && 'success' in json && json.success === false) { - const msg = typeof (json as { message?: unknown }).message === 'string' - ? (json as { message: string }).message - : 'komodo returned success=false'; - return { ok: false, kind: 'http-error', message: msg, url, httpStatus: res.status }; - } - } catch { - // Non-JSON body — trust the 2xx. - } - return { ok: true }; -} - -/** - * Create a new deployment entity in Komodo. Doesn't start the container — - * caller must follow up with `komodoDeploy(name)` to actually run it. - * - * The config shape here targets Komodo's DeploymentConfig: newline-delimited - * ports/volumes strings and an environment array with `{ variable, value }` - * entries. If Komodo rejects the payload, the error surfaces in the http-error - * message so we can iterate on the shape without guessing. - */ -export async function komodoCreateDeployment(spec: CreateDeploymentSpec): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - const body = { - name: spec.name, - config: { - server_id: spec.server, - image: { type: 'Image', params: { image: spec.image } }, - network: spec.network ?? 'komodo_default', - ports: spec.ports.join('\n'), - volumes: spec.volumes.join('\n'), - environment: spec.environment, - restart: 'unless-stopped' - } - }; - return postAndClassify('write/CreateDeployment', body); -} - -/** Start (or restart) the container backing a Komodo deployment. */ -export async function komodoDeploy(deployment: string): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - return postAndClassify('execute/Deploy', { deployment }); -} - -/** Runtime controls use the saved Komodo resource, never the EC display name. */ -export async function komodoRuntimeAction(server: string, action: 'start' | 'stop'): Promise { - if (!komodoConfigured()) return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - const row = getServer(server); - if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' }; - const type = row.komodo_type ?? 'deployment'; - const verb = action === 'start' ? 'Start' : 'Stop'; - if (type === 'container') { - if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' }; - return postAndClassify(`execute/${verb}Container`, { server: row.komodo_server, container: row.komodo_name }); - } - if (type === 'stack') return postAndClassify(`execute/${verb}Stack`, { stack: row.komodo_name }); - return postAndClassify(`execute/${verb}Deployment`, { deployment: row.komodo_name }); -} - -/** Stop + remove the container backing a Komodo deployment. Entity remains. */ -export async function komodoDestroyDeployment(deployment: string): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - return postAndClassify('execute/DestroyDeployment', { deployment }); -} - -/** Remove the deployment entity from Komodo entirely. */ -export async function komodoDeleteDeployment(deployment: string): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - // Komodo's DeleteDeployment historically accepts `{ id }` where id may be - // the name; supply both to be forgiving. - return postAndClassify('write/DeleteDeployment', { id: deployment, name: deployment }); -} - -// ---------- Tags ---------- - -export type KomodoTagResult = - | { ok: true; id: string } - | { ok: false; message: string }; - -// Accept API string IDs and raw Mongo Extended JSON: { _id: { $oid: "..." } }. -function extractTagId(t: unknown): string | null { - if (!t || typeof t !== 'object') return null; - const o = t as { id?: unknown; _id?: unknown }; - if (typeof o.id === 'string' && o.id.length > 0) return o.id; - if (typeof o._id === 'string' && o._id.length > 0) return o._id; - if (o._id && typeof o._id === 'object') { - const oid = (o._id as { $oid?: unknown }).$oid; - if (typeof oid === 'string' && oid.length > 0) return oid; - } - return null; -} - -// ListTags response shape varies by Komodo version: bare array `[...]`, or -// wrapped under one of a few common keys. Peel back the common wrappers. -function unwrapTagList(body: unknown): unknown[] { - if (Array.isArray(body)) return body; - if (body && typeof body === 'object') { - for (const key of ['tags', 'data', 'results', 'items']) { - const val = (body as Record)[key]; - if (Array.isArray(val)) return val; - } - } - return []; -} - -/** - * Fetch the tag list from Komodo. Two request styles are attempted in order: - * - * 1. `POST /read/ListTags` with an empty body — the per-endpoint style - * already used by the rest of this file (see execute/RestartDeployment). - * 2. `POST /read` with `{type: "ListTags", params: {}}` — the tagged-union - * style used by mbecker20/komodo's canonical API. If your Komodo build - * wires up only the tagged-union path, style #1 returns something we - * can't parse (empty array or non-tag payload) and we fall through. - * - * The winning parsed list is returned. The raw response bodies get logged - * only when both attempts fail to yield a match, so a fresh Komodo API shape - * is diagnosable without spamming stderr on the happy path. - */ -async function fetchTagsWithFallback(): Promise<{ tags: unknown[]; rawBodies: string[] }> { - const attempts: Array<{ path: string; body: unknown }> = [ - { path: 'read/ListTags', body: {} }, - { path: 'read', body: { type: 'ListTags', params: {} } } - ]; - const rawBodies: string[] = []; - for (const attempt of attempts) { - let res: Response; - try { - res = await komodoCall(attempt.path, attempt.body); - } catch (err) { - rawBodies.push(`${attempt.path}: network error — ${(err as Error).message}`); - continue; - } - if (!res.ok) { - const snippet = await res.text().catch(() => ''); - rawBodies.push(`${attempt.path}: HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`); - continue; - } - const raw = await res.text().catch(() => ''); - rawBodies.push(`${attempt.path}: ${truncate(raw)}`); - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - continue; - } - const tags = unwrapTagList(parsed); - if (tags.length > 0) return { tags, rawBodies }; - } - return { tags: [], rawBodies }; -} - -async function findTagByName(name: string): Promise { - const { tags, rawBodies } = await fetchTagsWithFallback(); - for (const t of tags) { - if (!t || typeof t !== 'object') continue; - const tagName = (t as { name?: unknown }).name; - if (typeof tagName !== 'string' || tagName !== name) continue; - const id = extractTagId(t); - if (id) return { ok: true, id }; - } - // Miss — dump the raw bodies so the next iteration knows the real shape. - console.warn( - `[komodoEnsureTag] no matchable tag "${name}" in any ListTags variant. Bodies:\n ` + - rawBodies.join('\n ') - ); - return null; -} - -/** - * Resolve a tag by name, creating it if it doesn't exist. Returns the tag's - * Komodo id (Mongo ObjectId string) for use in UpdateResourceMeta. - * - * Best-effort: callers should treat failure as "skip tagging", not "roll back - * the whole operation". Tags are cosmetic organization inside Komodo. - */ -export async function komodoEnsureTag(name: string): Promise { - if (!komodoConfigured()) return { ok: false, message: 'komodo not configured' }; - - // 1. Look for an existing tag by name. - const existing = await findTagByName(name); - if (existing) return existing; - - // 2. Not found — create it. - let createRes: Response; - try { - createRes = await komodoCall('write/CreateTag', { name }); - } catch (err) { - return { ok: false, message: `CreateTag ${name}: ${(err as Error).message}` }; - } - if (createRes.ok) { - const body = await createRes.json().catch(() => null); - const id = extractTagId(body); - if (id) return { ok: true, id }; - // 200 but shape we don't recognize — fall back to re-list rather than - // fail, since the tag definitely exists now. - const fresh = await findTagByName(name); - if (fresh) return fresh; - console.warn(`[komodoEnsureTag] CreateTag ${name}: unexpected 2xx body, re-list also missed`, body); - return { ok: false, message: `CreateTag ${name}: unexpected response shape` }; - } - - // Non-2xx. Duplicate-key 500 means the tag exists under a different - // ListTags projection than we probed — re-list and try to find it. If - // that still misses, surface the create error. - const snippet = await createRes.text().catch(() => ''); - const looksLikeDuplicate = - createRes.status === 500 && - (/duplicate key/i.test(snippet) || /E11000/i.test(snippet) || /dup key/i.test(snippet)); - if (looksLikeDuplicate) { - const fresh = await findTagByName(name); - if (fresh) return fresh; - return { - ok: false, - message: `CreateTag ${name}: duplicate reported but re-list missed it. Body: ${truncate(snippet)}` - }; - } - return { - ok: false, - message: `CreateTag ${name}: HTTP ${createRes.status}${snippet ? ` — ${truncate(snippet)}` : ''}` - }; -} - -/** - * Attach tags to a Komodo resource. `tagIds` is the array of tag Mongo IDs - * from `komodoEnsureTag`. Replaces the resource's existing tag set — pass the - * full desired list, not a delta. - * - * Komodo's target.id accepts either the true id or the resource name for - * Deployments, so we pass the name here to save a lookup. - */ -export async function komodoUpdateTagsOnResource( - targetType: 'Deployment' | 'Stack' | 'Server' | 'Repo' | 'Build' | 'Procedure', - targetName: string, - tagIds: string[] -): Promise { - if (!komodoConfigured()) { - return { ok: false, kind: 'unconfigured', message: 'komodo not configured' }; - } - // UpdateResourceMeta owns tags; omit the other optional metadata fields - // so the deployment's description and template flag remain unchanged. - return postAndClassify('write/UpdateResourceMeta', { - target: { type: targetType, id: targetName }, - tags: tagIds - }); -} - -// ---------- Extractor ---------- - -// Property names Komodo/Docker use for the collection of port mappings. -const PORT_ARRAY_KEYS = new Set(['ports', 'Ports', 'port_bindings', 'PortBindings']); -// Value field names for the host-facing port on an object entry. -const HOST_PORT_KEYS = ['published', 'PublishedPort', 'public_port', 'PublicPort', 'HostPort', 'hostPort', 'host_port']; - -interface PortsCandidate { - arr: unknown[]; - /** True when the array was reached via a `ports`-flavored key — preferred. */ - preferred: boolean; -} - -/** - * Walks a Komodo response looking for a UDP port. Handles the shapes we've - * seen in the wild: - * - * docker-compose object: [{ protocol: "udp", published: 4976, target: 28960 }] - * docker-compose caps: [{ Protocol: "udp", PublishedPort: 4976 }] - * docker-run string: ["4976:28960/udp", "3074:3074/tcp"] - * host-network string: ["28962/udp"] - * docker-inspect object: { "28960/udp": [{ "HostIp": "0.0.0.0", "HostPort": "28962" }] } - * - * Arrays reached via a `ports`/`Ports`/`port_bindings`/`PortBindings` key - * are preferred over arrays discovered elsewhere in the tree, so an - * unrelated array (e.g. tags, labels) can't shadow a real port list. - */ -export function extractUdpPort(payload: unknown): number | null { - const candidates: PortsCandidate[] = []; - // Objects whose keys encode `/` — a la docker inspect. - const containerKeyed: Record[] = []; - const seen = new Set(); - - const walk = (v: unknown, viaPortsKey: boolean): void => { - if (v === null || typeof v !== 'object' || seen.has(v)) return; - seen.add(v); - if (Array.isArray(v)) { - for (const item of v) walk(item, false); - candidates.push({ arr: v, preferred: viaPortsKey }); - return; - } - const obj = v as Record; - // Detect docker-inspect container-keyed objects: any key like `28960/udp`. - for (const k of Object.keys(obj)) { - if (/^\d+\/(udp|tcp)$/i.test(k)) { - containerKeyed.push(obj); - break; - } - } - for (const [key, val] of Object.entries(obj)) { - // Komodo's GetDeployment renders `config.ports` as a single - // newline-delimited string ("28963:28963/udp\n28963:28963/tcp"). - // Synthesize an array from it and treat as a preferred candidate. - if (PORT_ARRAY_KEYS.has(key) && typeof val === 'string' && val.trim()) { - const parts = val - .split(/[\r\n,;]+/) - .map((s) => s.trim()) - .filter(Boolean); - if (parts.length > 0) { - candidates.push({ arr: parts, preferred: true }); - } - } - walk(val, PORT_ARRAY_KEYS.has(key)); - } - }; - walk(payload, false); - - // Sort so preferred `ports`-keyed arrays win over accidental hits. - candidates.sort((a, b) => (a.preferred === b.preferred ? 0 : a.preferred ? -1 : 1)); - - // Try object-entry shape first — most Komodo/Docker responses use this. - for (const { arr } of candidates) { - for (const entry of arr) { - if (entry && typeof entry === 'object' && !Array.isArray(entry)) { - const e = entry as Record; - const proto = String(e.protocol ?? e.Protocol ?? e.type ?? e.Type ?? '').toLowerCase(); - if (proto !== 'udp') continue; - for (const field of HOST_PORT_KEYS) { - const raw = e[field]; - if (raw === undefined || raw === null) continue; - const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw); - if (Number.isFinite(num) && num > 0) return num; - } - } - } - } - - // Then string shapes: "host:container/udp" or "container/udp" (host-net). - for (const { arr } of candidates) { - for (const entry of arr) { - if (typeof entry !== 'string') continue; - const hostMapped = /^(\d+):\d+\/udp$/i.exec(entry); - if (hostMapped) return Number.parseInt(hostMapped[1], 10); - const hostNet = /^(\d+)\/udp$/i.exec(entry); - if (hostNet) return Number.parseInt(hostNet[1], 10); - } - } - - // Finally, docker-inspect container-keyed shape: - // { "28960/udp": [{ HostIp, HostPort }] } - for (const obj of containerKeyed) { - for (const [key, val] of Object.entries(obj)) { - if (!/^\d+\/udp$/i.test(key)) continue; - if (!Array.isArray(val)) continue; - for (const binding of val) { - if (!binding || typeof binding !== 'object') continue; - const b = binding as Record; - for (const field of HOST_PORT_KEYS) { - const raw = b[field]; - if (raw === undefined || raw === null) continue; - const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw); - if (Number.isFinite(num) && num > 0) return num; - } - } - } - } - - return null; -} diff --git a/src/lib/server/integrations/netProbe.ts b/src/lib/server/integrations/netProbe.ts deleted file mode 100644 index da5ca12..0000000 --- a/src/lib/server/integrations/netProbe.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { createConnection } from 'node:net'; -import { lookup } from 'node:dns/promises'; - -export type TcpProbeOutcome = - | 'connected' - | 'refused' - | 'timeout' - | 'dns-fail' - | 'unreachable' - | 'error'; - -export interface TcpProbeResult { - label: string; - host: string; - port: number; - resolvedIp: string | null; - outcome: TcpProbeOutcome; - elapsedMs: number; - detail?: string; -} - -const DEFAULT_TIMEOUT_MS = 3000; - -/** - * Best-effort DNS resolution. Returns null on failure so the caller can - * distinguish "DNS failed" from "connect failed". - */ -async function resolve(host: string): Promise<{ ip: string | null; error: string | null }> { - try { - const r = await lookup(host); - return { ip: r.address, error: null }; - } catch (err) { - const e = err as NodeJS.ErrnoException; - return { ip: null, error: `${e.code ?? 'UNKNOWN'}: ${e.message}` }; - } -} - -/** - * Attempt a raw TCP connect to `host:port` with an explicit timeout, and - * classify the result. Purely diagnostic — no bytes are sent after the - * handshake completes. Isolates network reachability from every HTTP-layer - * concern (auth, TLS, redirects, wrong path). - */ -export async function tcpProbe( - label: string, - host: string, - port: number, - timeoutMs: number = DEFAULT_TIMEOUT_MS -): Promise { - const started = Date.now(); - const { ip, error: dnsError } = await resolve(host); - - if (!ip) { - return { - label, - host, - port, - resolvedIp: null, - outcome: 'dns-fail', - elapsedMs: Date.now() - started, - detail: dnsError ?? 'DNS lookup returned no address' - }; - } - - return new Promise((resolve) => { - const sock = createConnection({ host: ip, port, timeout: timeoutMs }); - let settled = false; - const finish = (outcome: TcpProbeOutcome, detail?: string): void => { - if (settled) return; - settled = true; - try { sock.destroy(); } catch { /* already destroyed */ } - resolve({ label, host, port, resolvedIp: ip, outcome, elapsedMs: Date.now() - started, detail }); - }; - sock.once('connect', () => finish('connected')); - sock.once('timeout', () => finish('timeout', `no SYN-ACK within ${timeoutMs}ms`)); - sock.once('error', (err) => { - const code = (err as NodeJS.ErrnoException).code ?? ''; - const msg = (err as Error).message ?? String(err); - if (code === 'ECONNREFUSED') return finish('refused', code); - if (code === 'EHOSTUNREACH' || code === 'ENETUNREACH') return finish('unreachable', `${code}: ${msg}`); - return finish('error', `${code || 'unknown'}: ${msg}`); - }); - }); -} - -export async function tcpProbeAll( - trials: Array<{ label: string; host: string; port: number }>, - timeoutMs: number = DEFAULT_TIMEOUT_MS -): Promise { - // Run in parallel — they don't share sockets and the whole point is to - // finish fast even when some probes hit the timeout. - return Promise.all(trials.map((t) => tcpProbe(t.label, t.host, t.port, timeoutMs))); -} diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index a32e982..869b413 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -5,7 +5,7 @@ import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { latestGameSubmission } from '../fleet/status-evidence.js'; import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js'; import { rconPlutoniumVersion } from './rcon.js'; -import { komodoInspectHealthy } from './komodo.js'; +import { nativeHealth } from '../backends/native.js'; import { restartRecoveryPending } from '../provisioning/index.js'; import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js'; import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js'; @@ -60,7 +60,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool const epoch = latestGameSubmission(server); const recovering = recoveryBlocked(); const health = !recovering && original.desired_state !== 'stopped' - ? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false }; + ? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false }; if (suppressed()) return null; let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' } : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); diff --git a/src/lib/server/integrations/rcon.ts b/src/lib/server/integrations/rcon.ts index 5b1fe30..c5539b5 100644 --- a/src/lib/server/integrations/rcon.ts +++ b/src/lib/server/integrations/rcon.ts @@ -1,9 +1,8 @@ import { createSocket } from 'node:dgram'; import type { Socket } from 'node:dgram'; -import { config, komodoConfigured, rconConfigured } from '../config.js'; -import { getServer, setServerField } from '../fleet/servers.js'; +import { config, rconConfigured } from '../config.js'; +import { getServer } from '../fleet/servers.js'; import { latestGameSubmission } from '../fleet/lifecycle-evidence.js'; -import { komodoResolveGamePort } from './komodo.js'; import { parsePlutoniumRevision } from '../../plutonium.js'; import { nativeInstance, nativeRequest } from '../backends/native.js'; @@ -81,27 +80,15 @@ async function sendRcon(host: string, port: number, command: string): Promise { const row = getServer(server); if (!row) return { endpoint: null, reason: 'unknown-server' }; - let port = row.game_port; - if (!port) { - if (!komodoConfigured()) { - return { endpoint: null, reason: 'no-port-no-komodo' }; - } - port = await komodoResolveGamePort(server); - if (port) setServerField(server, 'game_port', port); - } - if (!port) return { endpoint: null, reason: 'no-port-komodo-empty' }; - return { endpoint: { host: row.rcon_host ?? config.rcon.host, port } }; + if (!row.game_port) return { endpoint: null, reason: 'no-port' }; + return { endpoint: { host: row.rcon_host ?? config.rcon.host, port: row.game_port } }; } function stripPrefix(buf: Buffer): string | null { @@ -421,20 +408,7 @@ export async function rconStatusResult(server: string): Promise { if (observation.identity !== monitoringIdentity(server)) return; if (!scanSuppressed(server, 'health') && observation.healthy) { const current = active(server, 'health'); - if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified'); + if (current) transition(current.id, 'resolved', 'Fresh game status and host agent health verified'); } else if (!scanSuppressed(server, 'health')) { const problem = detect(server, 'health', observation.reason, observation); const startup = ['starting','restarting'].includes(observation.containerState ?? ''); @@ -334,8 +333,8 @@ async function run(forceScan: boolean): Promise { const lastScan = setting('watchdog_last_scan'); if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return; setSetting('watchdog_last_scan', iso()); - if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) { - setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return; + if (!listServers().some(server => nativeInstance(server.name))) { + setSetting('watchdog_scan_error', 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'); return; } const errors: string[] = []; const servers = listServers(); diff --git a/src/lib/server/pipeline/restart-rollout.ts b/src/lib/server/pipeline/restart-rollout.ts index a00e9f3..abe5b2d 100644 --- a/src/lib/server/pipeline/restart-rollout.ts +++ b/src/lib/server/pipeline/restart-rollout.ts @@ -1 +1 @@ -export { restartSettling, rolloutWaitReason, refreshRestartRollout, rolloutHealth } from '../provisioning/restarts.js'; +export { restartSettling, rolloutWaitReason, rolloutHealth } from '../provisioning/restarts.js'; diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 8e35d4a..976dcfe 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -100,8 +100,8 @@ function queueStatus(srv: ServerRow, probe?: () => ReturnType void; diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index d103079..b6fe8a6 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -121,7 +121,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } SELECT collection_id, deployment_id FROM server_collection_members UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined; - return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null, + return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null, canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null, workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'), activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources, @@ -133,7 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } })(); } -// This stage is written only after Komodo has confirmed deployment deletion. +// This stage is written only after the host agent has confirmed instance removal. export const REMOVING_SERVER = 'Removing server files and registry'; export function canCancelRemoval(id: string): boolean { @@ -154,15 +154,6 @@ export function cancelRemoval(id: string): boolean { })(); } -export function claimRemoval(id: string): boolean { - return db().prepare(`UPDATE server_deployments SET state = 'running' - WHERE id = ? AND action = 'destroy' AND state = 'waiting' - AND (server_deleted_at IS NULL OR (stage = ? - AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server) - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server - AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0; -} - export function canResumeCleanup(id: string): boolean { return !!db().prepare(`SELECT d.id FROM server_deployments d WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server)) @@ -177,7 +168,7 @@ export function resumeCleanup(id: string): boolean { preserveOutcome(id); db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id); recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null, - event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } }); + event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } }); return true; })(); } @@ -207,7 +198,7 @@ export function resumeRebuild(id: string): boolean { preserveOutcome(id); db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id); recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null, - event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } }); + event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } }); return true; })(); } @@ -218,7 +209,7 @@ export function resumeVerification(id: string): boolean { if (!canResumeVerification(id)) return false; preserveOutcome(id); db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id); - deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null, + deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null, event: { kind: 'resumed', message: 'Resumed verification of the existing server' } }); return true; })(); diff --git a/src/lib/server/provisioning/index.ts b/src/lib/server/provisioning/index.ts index 71d8932..05478c5 100644 --- a/src/lib/server/provisioning/index.ts +++ b/src/lib/server/provisioning/index.ts @@ -2,7 +2,7 @@ // Execution is driven exclusively by the provisioning worker. export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js'; export type { LifecycleOperation, RestartSource } from './requests.js'; -export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js'; +export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js'; export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js'; export { deploymentActivity } from './activity.js'; export { restartSettling, restartRecoveryPending } from './restarts.js'; diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index 428f8f7..acc92a7 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -1,47 +1,18 @@ -import { listHosts } from '../hosts/index.js'; -import { fleetSettings } from '../settings.js'; -import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js'; -import { assertProvisioningNotCancelled, provisioningCancelled } from './cancellation.js'; -import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js'; -import { randomUUID } from 'node:crypto'; -import type { LaunchPreview } from '$lib/deployment-plan.js'; -import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; -import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js'; import { db } from '../db/index.js'; -import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; -import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js'; -import { - komodoCreateDeployment, - komodoDeploy, - komodoDestroyDeployment, - komodoDeleteDeployment, - komodoInspectHealthy, - komodoEnsureTag, - komodoUpdateTagsOnResource, - type CreateDeploymentSpec, - type DeploymentEnvVar -} from '../integrations/komodo.js'; -import { MAP_CONFIG_FILE } from '../fleet/server-config.js'; -import { recordScript } from '../fleet/inventory.js'; -import { getVersion } from '../fleet/scripts.js'; -import type { ProvisioningBlocker } from '../../provisioning-detail.js'; -import { recordProgress, type ProgressUpdate } from './progress.js'; -import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js'; -import { nativeInstance, nativeDeployment } from '../backends/native.js'; +import type { GameMode, ZombieGame } from '../fleet/map-aliases.js'; +import { nativeInstance } from '../backends/native.js'; +import { beginDeployment, deploymentStage, resumeCleanup } from './history.js'; import { previewNativeCreation, queueNativeCreation, queueNativeRebuild, cancelQueuedNativeCreation } from './native.js'; export type CreateServerFailure = | 'bad-request' | 'conflict' - | 'unconfigured' // Komodo creds / target server missing + | 'unconfigured' // native host configuration missing | 'unknown-map' // slug isn't in the catalog | 'no-slot' // all lobby slots for this map are taken | 'no-port' // port range exhausted | 'fs' // couldn't create host dir - | 'komodo' // Komodo CreateDeployment / Deploy rejected | 'db'; // shouldn't happen, but wraps INSERT failures export type CreateServerResult = @@ -50,7 +21,7 @@ export type CreateServerResult = export type DestroyServerResult = | { ok: true; warnings: string[] } - | { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] }; + | { ok: false; kind: 'not-found' | 'conflict' | 'fs' | 'db'; message: string; warnings: string[] }; export interface CreateServerInput { hostId?: string; @@ -68,11 +39,6 @@ export interface CreateServerInput { createdBy?: string; // 'ui' by default } -/** - * Find the lowest lobby number `n` such that `t5___` isn't in - * the servers table. Caps at 20 to stay inside the port range — with 20 total - * ports across the whole fleet, no one map can hold more than that. - */ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null { const rows = db() .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION @@ -92,471 +58,9 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = [] return null; } -/** - * Lowest unused port in the configured range. Reads every server's game_port - * from the DB (which EC controls end-to-end under the new naming scheme, so - * the DB is authoritative — no need to ask Komodo). - */ -export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null { - const rows = db() - .prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION - SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`) - .all() as Array<{ name: string; game_port: number }>; - const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]); - for (let p = config.portRange.min; p <= config.portRange.max; p++) { - if (!taken.has(p)) return p; - } - return null; -} - -function buildEnv(args: { - game: ZombieGame; - name: string; - port: number; - plutoKey: string; -}): DeploymentEnvVar[] { - return [ - { variable: 'PLUTAINER_SERVER_NAME', value: args.name }, - { variable: 'PLUTAINER_PORT', value: String(args.port) }, - { variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' }, - { variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) }, - // Plutainer seeds/loads the main config first. Load our generated map - // config afterward, then rotate. A separate exec avoids quoted values - // in EXTRA_ARGS, which Plutainer splits on whitespace. - { variable: 'PLUTAINER_MAP_ROTATE', value: 'false' }, - // Give telemetry scripts the registry name before the first map starts. - { variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` }, - { variable: 'PLUTO_SERVER_KEY', value: args.plutoKey }, - { variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password } - ]; -} - -function buildSpec(args: { - game: ZombieGame; - name: string; - port: number; - hostDir: string; - plutoKey: string; - recipe: LaunchRecipe; -}): CreateDeploymentSpec { - return { - name: args.name, - server: config.komodo.targetServer, - image: args.recipe.image, - network: config.komodo.defaultNetwork, - // Container listens on PLUTAINER_PORT (== args.port), so host and - // container ports match — no NAT inside the bridge network. - ports: [ - `${args.port}:${args.port}/udp`, - `${args.port}:${args.port}/tcp` - ], - volumes: [ - `${args.hostDir}:/home/plutainer/app`, - `${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro` - ], - environment: buildEnv(args) - }; -} - -let creating = false; -async function observeCommand(operation: string, command: () => Promise, retainReceipt = false): Promise { - assertProvisioningNotCancelled(operation); - try { - const result = await command(); - if (!retainReceipt) assertProvisioningNotCancelled(operation); - return result; - } catch (err) { - if (provisioningCancelled(operation)) throw err; - recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } }); - throw err; - } -} -interface PreparedCreation { - ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry; -} - -interface PreviewRow { - id: string; expires_at: string; server: string | null; port: number | null; - recipe: string; blueprint_name: string | null; deployment_id: string | null; -} - -/** Read-only checks; a preview never reserves a slot or writes credentials. */ -export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview { - if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input); - if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.'); - let recipe: LaunchRecipePlan; - let blueprintName: string | null = null; - if (input.blueprintId !== undefined) { - if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.'); - ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); - } else if (input.deploymentId !== undefined) { - if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.'); - const source = recipeForDeployment(input.deploymentId); - if (!source) throw new Error('This deployment has no recorded launch recipe.'); - blueprintName = source.blueprintName; - // History describes the previous launch; only its map selection is reusable. - recipe = captureRecipePlan(source.recipe.game, source.recipe.mapSlug); - } else { - if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); - recipe = captureRecipePlan(input.game, input.mapSlug); - } - const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`); - const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`; - const port = nextAvailablePort(); - const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer); - const checks: LaunchPreview['checks'] = [ - { label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' }, - { label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' }, - { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' }, - { label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` }, - { label: 'Promoted configuration', state: 'pass', detail: recipe.configSource }, - { label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game files. This preview does not probe volumes, images, credentials, or network access.' } - ]; - const id = randomUUID(); - const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); - db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString()); - db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)') - .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); - return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, - checks, ready: checks.every(check => check.state !== 'blocked') }; -} - -// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it. -export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult { - const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null; - if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input); - return db().transaction((): CreateServerResult => { - const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined; - if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' }; - const recipe: LaunchRecipePlan | undefined = preview ? JSON.parse(preview.recipe) : undefined; - if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) { - return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' }; - } - // A repeated submit after a lost response returns the same operation. - if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id }; - if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' }; - if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) { - return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' }; - } - if (recipe && !config.rcon.password) { - return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' }; - } - const prepared = prepareCreation(input, releasingServers); - if (!prepared.ok) return prepared; - // A preview records choices, not version pins. Promotions remain authoritative. - const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug); - const storedRecipe = recipeStorage(currentRecipe); - const id = beginDeployment(prepared.name, 'create', input.createdBy); - db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary, blueprint_name) VALUES (?, ?, ?, ?)') - .run(id, storedRecipe.recipe, storedRecipe.recipeSummary, preview?.blueprint_name ?? null); - if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id); - db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id); - db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') - .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical); - deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null, - blocker: { code: 'startup-queue', message: 'Waiting for startup admission and available capacity.', automatic: true } }); - return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id }; - })(); -} - -/** Reserve the same identity and port before removing any part of the instance. */ -export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult { - if (nativeInstance(name)) return queueNativeRebuild(name, triggeredBy); - const blocker = activeBetaRemovalBlocker(name); - if (blocker) return { ok: false, kind: 'conflict', message: blocker }; - const server = getServer(name); - const game = gameOfServer(name); - const map = server?.map_slug && zombieMapBySlug(game, server.map_slug); - if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name - || (server.host_dir && server.host_dir !== serverDir(name))) { - return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' }; - } - if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' }; - if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` }; - // Validate that the replacement setup can be composed before destruction. - const recipe = captureRecipePlan(game, map.slug); - return db().transaction(() => { - const id = beginDeployment(name, 'rebuild', triggeredBy); - db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') - .run(id, name, server.game_port, 'zombie', map.slug, map.canonical); - db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id); - const storedRecipe = recipeStorage(recipe); - db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)') - .run(id, storedRecipe.recipe, storedRecipe.recipeSummary); - deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null, - blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } }); - return { ok: true as const, name, port: server.game_port!, deploymentId: id }; - })(); -} - -/** Caller has just verified emptiness and excluded conflicting work. */ -export async function rebuildServer(id: string): Promise { - const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined; - if (!job) return; - const blocker = activeBetaRemovalBlocker(job.server); - if (blocker) { - // Keep the cleanup checkpoint intact when resuming after remote deletion. - if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker }); - else deploymentStage(id, blocker, { phase: 'eligibility', blocker: betaRemovalEvidence(job.server, blocker), checks: [] }); - return; - } - if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) { - // The cleanup checkpoint must survive capacity waits and app restarts. - const progress: ProgressUpdate = { phase: job.stage === REMOVING_SERVER ? 'cleanup' : 'eligibility', - blocker: { code: 'startup-capacity', message: 'Waiting for a server creation slot before rebuilding', automatic: true } }; - if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding', progress); - else recordProgress(id, progress); - return; - } - // Recheck prerequisites while the original instance still exists. - if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; } - try { captureRecipePlan(gameOfServer(job.server), job.map_slug); } - catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; } - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id); - destroying.add(job.server); - try { - const result = job.stage === REMOVING_SERVER - ? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id); - if (provisioningCancelled(id)) return; - if (!result.ok || result.warnings.length) { - finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings }); - return; - } - db().transaction(() => { - db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id); - db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id); - deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', checks: [], - blocker: { code: 'startup-queue', message: 'Old instance removed; waiting for fresh-instance startup admission.', automatic: true }, - checkpoint: { key: 'cleanup', label: 'Old instance files and registry removed' } }); - })(); - } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); } - finally { destroying.delete(job.server); } -} - -function nextName(input: CreateServerInput): string | null { - const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`; - const lobby = nextLobbyNumber(base); - return lobby === null ? null : `${base}_${lobby}`; -} - -// Internal callers use the same durable queue and capacity limit as HTTP launches. -// Await this pass's provisioning when possible; a busy host leaves the job queued. -export async function createServer(input: CreateServerInput): Promise { - const queued = queueCreateServer(input); - if (!queued.ok) return queued; - const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string }; - if (existing.state !== 'queued') return queued; - const outcomes = await creationPass(); - const outcome = outcomes.get(queued.deploymentId); - if (outcome instanceof Error) throw outcome; - return outcome ?? queued; -} - -function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract { - const game = input.game ?? 't5'; - if (game !== 't5' && game !== 't6') { - return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' }; - } - if (input.gameMode !== 'zombie') { - return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` }; - } - if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) { - return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' }; - } - if (!config.komodo.targetServer) { - return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' }; - } - - const mapEntry = zombieMapBySlug(game, input.mapSlug); - if (!mapEntry) { - return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` }; - } - - const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`; - const lobby = nextLobbyNumber(baseName); - if (lobby === null) { - return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` }; - } - const name = `${baseName}_${lobby}`; - if (input.expectedName !== undefined && input.expectedName !== name) { - return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' }; - } - const plutoKey = input.plutoKey?.trim() || getServerKey(name); - if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` }; - - const port = nextAvailablePort([], releasingServers); - if (port === null) { - return { - ok: false, - kind: 'no-port', - message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first` - }; - } - - // Save before external operations so a failed deployment can also be retried. - try { saveServerKey(name, plutoKey); } - catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; } - - return { ok: true, name, port, plutoKey, mapEntry }; -} - -async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise { - const { name, port, plutoKey, mapEntry } = prepared; - const game = gameOfServer(name); - // Resolve once more at provisioning so promotions made while queued also apply. - // Persist the exact attempted files for history, never as input to future launches. - const recipe = captureRecipe(game, mapEntry.slug); - const storedRecipe = recipeStorage(recipe); - db().transaction(() => { - db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?) - ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe, recipe_summary = excluded.recipe_summary`) - .run(operation, storedRecipe.recipe, storedRecipe.recipeSummary); - deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [], - checkpoint: { key: 'startup-queue', label: 'Admitted to startup' }, - event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } }); - })(); - const configFile = recipe.configFile; - const hostDir = serverDir(name); - - // Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly - // after — plutainer expects to be able to write to /home/plutainer/app. - try { - await mkdir(hostDir, { recursive: true, mode: 0o777 }); - await chmod(hostDir, 0o777); - for (const file of recipe.files) { - const dest = destOfKind(name, file.kind); - const target = join(dest, file.path); - await mkdir(dirname(target), { recursive: true, mode: 0o777 }); - // Make every newly created ancestor writable by Plutainer, even - // when the manager's umask removed write permissions at mkdir. - for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) { - await chmod(dir, 0o777); - } - await writeFile(target, Buffer.from(file.content, 'base64')); - await chmod(target, 0o666); - } - } catch (err) { - return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` }; - } - - assertProvisioningNotCancelled(operation); - // Step 2: Komodo CreateDeployment. - deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment', - checkpoint: { key: 'prepare', label: 'Server files prepared' }, - command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } }); - const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe }); - db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), operation); - // A confirmed creation must retain its identity for inspection/removal even if cancelled in flight. - const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec), true); - if (!createRes.ok && provisioningCancelled(operation)) assertProvisioningNotCancelled(operation); - if (!createRes.ok) { - recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } }); - await rm(hostDir, { recursive: true, force: true }).catch(() => {}); - return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` }; - } - recordProgress(operation, { command: { state: 'accepted', message: 'Komodo created the deployment.' }, - checkpoint: { key: 'remote-deployment-created', phase: 'create-deployment', label: 'Komodo deployment created' } }); - - // Step 3: DB row — insert *before* execute/Deploy so the sweep sees the - // server immediately even if Deploy hangs. - try { - db().transaction(() => { - db().prepare( - `INSERT INTO servers - (name, komodo_type, komodo_name, game, map, - game_mode, map_slug, host_dir, config_file, game_id, image, - game_port, created_by, container_state) - VALUES - (?, 'deployment', ?, ?, ?, - ?, ?, ?, ?, ?, ?, - ?, ?, 'starting')` - ).run( - name, name, game, mapEntry.canonical, - input.gameMode, mapEntry.slug, hostDir, - configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image, - port, input.createdBy ?? 'ui' - ); - for (const file of recipe.files) { - const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null; - recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create'); - } - })(); - } catch (err) { - // Roll back Komodo + FS if the DB insert fails (very unlikely — most - // often a race where the same name got inserted twice). - await komodoDestroyDeployment(name).catch(() => {}); - await komodoDeleteDeployment(name).catch(() => {}); - await rm(hostDir, { recursive: true, force: true }).catch(() => {}); - return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` }; - } - - assertProvisioningNotCancelled(operation); - // Step 4: apply Komodo tags so operators can filter the deployment list - // by game/map. Best-effort — a tag failure doesn't roll back the server; - // the deployment is fully functional without cosmetic tags. Fires in - // parallel to keep the create latency down. - void applyDeploymentTags(name, game, mapEntry.canonical); - - // Step 5: execute/Deploy — actually start the container. - db().transaction(() => { - deploymentStage(operation, 'Starting container', { phase: 'container-action', - checkpoint: { key: 'create-deployment', label: 'Deployment created and server registered' }, - command: { state: 'attempted', message: 'Container startup submission attempted.' } }); - db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation); - })(); - const deployRes = await observeCommand(operation, () => komodoDeploy(name)); - if (!deployRes.ok) { - if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) { - // A timeout or server error may arrive after the remote boot began. - // Preserve files, identity, and capacity until readiness is known. - recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message); - recordProgress(operation, { command: { state: 'uncertain', message: 'Startup may have begun: ' + deployRes.message } }); - return { ok: true, name, port, deploymentId: operation }; - } - // Container never started — undo everything so retrying is clean. - recordProgress(operation, { command: { state: 'rejected', message: 'Startup request rejected: ' + deployRes.message } }); - await komodoDeleteDeployment(name).catch(() => {}); - db().prepare('DELETE FROM servers WHERE name = ?').run(name); - await rm(hostDir, { recursive: true, force: true }).catch(() => {}); - return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` }; - } - - recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted startup; readiness still requires verification.' }, - checkpoint: { key: 'container-action', label: 'Container startup accepted' } }); - return { ok: true, name, port, deploymentId: operation }; -} - -/** - * Ensure `game` and `map` tags exist in Komodo, then attach both to the given - * deployment. Purely cosmetic — any failure here logs and returns; the caller - * treats it as fire-and-forget. - */ -async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise { - try { - const [gameTag, mapTag] = await Promise.all([ - komodoEnsureTag(gameTagName), - komodoEnsureTag(mapTagName) - ]); - const tagIds: string[] = []; - if (gameTag.ok) tagIds.push(gameTag.id); - else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`); - if (mapTag.ok) tagIds.push(mapTag.id); - else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`); - if (tagIds.length === 0) return; - - const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds); - if (!res.ok) { - console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`); - } - } catch (err) { - console.warn(`[createServer] tag application for ${deployment} threw:`, err); - } -} - -const destroying = new Set(); +export function previewCreation(input: Parameters[0]) { return previewNativeCreation(input); } +export function queueCreateServer(input: CreateServerInput): CreateServerResult { return queueNativeCreation(input); } +export const queueRebuildServer = (name: string, triggeredBy = 'ui') => queueNativeRebuild(name, triggeredBy); // Persist the request before returning so navigation and client disconnects cannot lose it. function hasActiveCreation(name: string): boolean { @@ -577,6 +81,7 @@ export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerRe const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined; if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id }; if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; + if (!nativeInstance(name)) return { ok: false, kind: 'conflict', message: 'Server has no native host instance.', warnings: [] }; const operation = db().transaction(() => { const id = beginDeployment(name, 'destroy', triggeredBy); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); @@ -587,277 +92,6 @@ export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerRe return { ok: true, warnings: [], deploymentId: operation }; } -let destructionTick: Promise | null = null; -export function processDestructions(): Promise { - if (destructionTick) return destructionTick; - destructionTick = runDestructions().finally(() => { destructionTick = null; }); - return destructionTick; -} - -function betaRemovalEvidence(server: string, message: string): ProvisioningBlocker { - const beta = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') ORDER BY b.id LIMIT 1`) - .get(server) as { id: number } | undefined; - return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' }; -} - -interface DestructionJob { id: string; server: string; stage: string; server_deleted_at: string | null } - -async function runDestructions(): Promise { - const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' - AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at, rowid`).all(REMOVING_SERVER) as DestructionJob[]; - const active = new Set>(); - let next = 0; - try { - while (next < queued.length || active.size) { - // Recheck the saved limit before each admission. Lowering it never - // interrupts an active removal; its slot lasts through local cleanup. - while (next < queued.length && !creating && destroying.size < fleetSettings().destroyConcurrency - && active.size < fleetSettings().destroyConcurrency) { - const task = processDestruction(queued[next++]).finally(() => { active.delete(task); }); - active.add(task); - } - if (!active.size) break; // Creation or rebuilding holds the remaining capacity. - await Promise.race(active); - } - } finally { - // Keep the shared pass locked until every admitted removal has settled, - // including when an unexpected persistence error escapes a task. - await Promise.allSettled(active); - } -} - -async function processDestruction(operation: DestructionJob): Promise { - if (nativeDeployment(operation.id)) return; - if (creating || destroying.has(operation.server)) return; - if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return; - destroying.add(operation.server); - try { - if (operation.stage === REMOVING_SERVER) { - if (operation.server_deleted_at && getServer(operation.server)) { - finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' }); - return; - } - const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' - AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id); - if (newer) { - finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' }); - return; - } - if (!claimRemoval(operation.id)) return; - finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id)); - return; - } - if (!getServer(operation.server)) { - if (hasActiveCreation(operation.server)) { - const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL - AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined; - deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [], - blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true, - href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } }); - return; - } - finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' }); - return; - } - const blocker = activeBetaRemovalBlocker(operation.server); - if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); return; } - // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness. - const evidence = await probePlayerEvidence(operation.server); - let players = evidence.players; - let occupancy: NonNullable[number] = { key: 'occupancy', label: 'Player occupancy', - result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source, - checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }; - if (players === null) { - const health = await komodoInspectHealthy(operation.server); - if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) { - players = 0; - occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null }; - } - } - if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return; - // Beta ownership may have changed while the network probes were in flight. - const latestBlocker = activeBetaRemovalBlocker(operation.server); - if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); return; } - if (players !== 0) { - const message = players === null - ? 'Waiting for player count: server could not be reached' - : 'Waiting for server to empty: ' + players + ' player(s) online'; - deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy], - blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } }); - return; - } - if (!claimRemoval(operation.id)) return; - deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null, - checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } }); - const result = await destroyServerUnlocked(operation.server, operation.id); - finishDeployment(operation.id, result); - } catch (err) { - const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string }; - if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, { - phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true } - }); - else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message }); - } finally { destroying.delete(operation.server); } -} - -async function destroyServerUnlocked(name: string, operation: string): Promise { - const blocker = activeBetaRemovalBlocker(name); - if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; - const row = getServer(name); - if (!row) { - return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] }; - } - - const warnings: string[] = []; - - // 1: stop + remove the container. execute/DestroyDeployment queues the - // teardown; the response comes back before the container is actually gone. - deploymentStage(operation, 'Stopping and removing container', { phase: 'remove-container', blocker: null, checks: [], - checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' }, - command: { state: 'attempted', message: 'Container removal submission attempted.' } }); - const destroyRes = await observeCommand(operation, () => komodoDestroyDeployment(name)); - if (!destroyRes.ok) { - recordProgress(operation, { command: { state: destroyRes.kind === 'unreachable' || (destroyRes.httpStatus !== undefined && destroyRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: destroyRes.message } }); - return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings }; - } - recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted teardown; removal is not yet confirmed.' } }); - - // 2: wait for the container to actually disappear. Komodo refuses to - // DeleteDeployment while a container still exists for it, so poll the - // health probe until it stops reporting a running state. `ok:false` here - // is the *success* signal — it means the container is no longer inspectable. - const pollDeadline = Date.now() + 30_000; - while (Date.now() < pollDeadline) { - const health = await komodoInspectHealthy(name); - assertProvisioningNotCancelled(operation); - if (!health.ok) break; - await new Promise((resolve) => setTimeout(resolve, 1500)); - } - - // 3: remove the deployment entity itself. - deploymentStage(operation, 'Deleting Komodo deployment', { phase: 'delete-deployment', - command: { state: 'attempted', message: 'Komodo deployment deletion attempted.' } }); - const deleteRes = await observeCommand(operation, () => komodoDeleteDeployment(name)); - if (!deleteRes.ok) { - recordProgress(operation, { command: { state: deleteRes.kind === 'unreachable' || (deleteRes.httpStatus !== undefined && deleteRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: deleteRes.message } }); - return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings }; - } - - deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [], - command: { state: 'accepted', message: 'Komodo confirmed deployment deletion.' }, - checkpoints: [{ key: 'remove-container', label: 'Container removal confirmed by deployment deletion' }, - { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' }] }); - - return cleanupDestroyedServer(name, operation); -} - -async function cleanupDestroyedServer(name: string, operation: string): Promise { - assertProvisioningNotCancelled(operation); - const blocker = activeBetaRemovalBlocker(name); - if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; - const row = getServer(name); - const warnings: string[] = []; - // Persist the checkpoint before local work so failed cleanup can resume without RCON. - // Legacy operations already recorded this same stage after Komodo deletion. - deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [], - checkpoint: { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' } }); - // Host directory. Fall back to derived path if host_dir wasn't set on - // legacy rows. - const hostDir = row?.host_dir ?? serverDir(name); - try { - await rm(hostDir, { recursive: true, force: true }); - assertProvisioningNotCancelled(operation); - recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } }); - } catch (err) { - warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`); - } - - assertProvisioningNotCancelled(operation); - // The registry trigger preserves history and marks its old server identity. - // Live inventory, rollback snapshots, and restart slots still cascade away. - try { - db().transaction(() => { - const recovery = db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(name) as { operation_id: string | null } | undefined; - db().prepare('DELETE FROM servers WHERE name = ?').run(name); - if (recovery?.operation_id) recordProgress(recovery.operation_id, { recovery: { state: 'released', operationId: recovery.operation_id, - message: 'Recovery capacity released when the old instance registry was removed.' } }); - recordProgress(operation, { checkpoint: { key: 'registry-removed', label: 'Server registration removed', phase: 'cleanup' } }); - if (!warnings.length) recordProgress(operation, { checkpoint: { key: 'cleanup', label: 'Local files and registry cleanup completed' } }); - })(); - } catch (err) { - return { - ok: false, - kind: 'db', - message: `failed to remove DB rows for ${name}: ${(err as Error).message}`, - warnings - }; - } - - const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[]; - for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create' - ? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' }); - return { ok: true, warnings }; -} -interface CreationJob { - deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string; - submitted_at?: string | null; -} - -async function provisionCreation(id: string): Promise { - const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob; - creating = true; - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id); - try { - const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug); - const plutoKey = getServerKey(job.server); - if (!mapEntry || !plutoKey) { - const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' }; - finishDeployment(id, result); - return result; - } - const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by }, - { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id); - if (provisioningCancelled(id) && result.ok) return result; - if (!result.ok) finishDeployment(id, result); - else { - db().transaction(() => { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); - deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', checks: [], - blocker: { code: 'container-readiness', message: 'Waiting for container readiness.', automatic: true } }); - })(); - } - return result; - } catch (err) { - if (provisioningCancelled(id)) throw err; - const server = getServer(job.server); - if (server?.game_port === job.port) { - // A lost response may have happened after Komodo started the server. - // Keep its slot and observe readiness instead of starting another boot. - db().transaction(() => { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); - recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message); - deploymentStage(id, 'Verifying existing server after interrupted startup', { phase: 'verify-container', checks: [], - command: { state: 'uncertain', message: 'Startup response interrupted: ' + (err as Error).message }, - blocker: { code: 'container-readiness', message: 'Observing the existing instance after an uncertain startup.', automatic: true } }); - })(); - } else finishDeployment(id, { ok: false, message: (err as Error).message }); - throw err; - } finally { creating = false; } -} - -type CreationOutcomes = Map; -let creationTick: Promise | null = null; -function creationPass(): Promise { - if (creationTick) return creationTick; - creationTick = runCreations().finally(() => { creationTick = null; }); - return creationTick; -} - -export async function processCreations(): Promise { - await creationPass(); -} - /** One host-wide FIFO, shared by collections and individual launches. */ export function creationQueue() { type Entry = { id: string; server: string; state: string; stage: string }; @@ -866,184 +100,3 @@ export function creationQueue() { ORDER BY started_at, rowid`).all() as Entry[]; return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') }; } - -function creationBlocker(id: string): ProvisioningBlocker | null { - const dependency = db().prepare(`SELECT d.id, d.server, d.state FROM server_creation_dependencies dep - JOIN server_deployments d ON d.id = dep.removal_id - WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { id: string; server: string; state: string } | undefined; - if (dependency) return { code: 'removal-dependency', message: `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`, - automatic: true, href: `/provisioning/${encodeURIComponent(dependency.id)}`, label: 'Required removal' }; - const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port - WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined; - return occupied ? { code: 'port-occupied', message: `Waiting for port to be released by ${occupied.name}`, automatic: true, - href: `/servers/${encodeURIComponent(occupied.name)}`, label: 'Server using this port' } : null; -} - -function stillCheckingCreation(id: string): boolean { - return !!db().prepare(`SELECT 1 FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id - JOIN servers s ON s.name = j.server AND s.game_port = j.port - WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state = 'awaiting-health'`).get(id); -} - -async function runCreations(): Promise { - const outcomes: CreationOutcomes = new Map(); - if (creating || destroying.size > 0) { - for (const job of creationQueue().queued) recordProgress(job.id, { phase: 'startup-queue', blocker: { - code: 'startup-worker-busy', message: 'Waiting for in-flight instance creation or removal to finish.', automatic: true - } }); - return outcomes; - } - // Restart recovery is observation-only: never repeat external creation or - // deployment. A known interrupted startup must retain its readiness gate. - const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[]; - for (const { id } of interrupted) { - if (nativeDeployment(id)) continue; - if (resumeVerification(id)) continue; - // A queued removal does not make the interrupted container disappear. - // Keep its slot until removal finishes, without resuming normal work or - // confusing it with a newer creation of the same server identity. - const removing = db().prepare(`SELECT 1 FROM server_deployments d - JOIN server_creation_jobs j ON j.deployment_id = d.id - JOIN servers s ON s.name = j.server AND s.game_port = j.port - WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server - AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running')) - AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server - AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id); - if (removing) { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id); - deploymentStage(id, 'Waiting for requested server removal after interrupted startup', { checks: [], - blocker: { code: 'removal-pending', message: 'Waiting for the requested removal; the interrupted startup still occupies capacity.', automatic: true } }); - } - } - - // Check only previously started servers. Newly launched servers keep their - // slot until a later pass, even if a health probe would answer immediately. - const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[]; - for (const job of checking) { - const id = job.deployment_id; - if (nativeDeployment(id)) continue; - if (getServer(job.server)?.game_port !== job.port) { - finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' }); - continue; - } - const removal = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server) as { id: string } | undefined; - if (removal) { - recordProgress(id, { checks: [], blocker: { code: 'removal-pending', message: 'Creation verification is held while server removal is pending.', automatic: true, - href: `/provisioning/${encodeURIComponent(removal.id)}`, label: 'Removal operation' } }); - continue; - } - const checks: NonNullable = []; - let verifyingPlutonium = false; - try { - const komodo = await komodoInspectHealthy(job.server); - if (!stillCheckingCreation(id)) continue; - checks.push({ key: 'container', label: 'Container readiness', source: 'komodo', result: komodo.ok ? 'pass' : 'waiting', - checkedAt: new Date().toISOString(), actual: komodo.state ?? null, message: komodo.ok ? null : komodo.reason }); - if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason, { phase: 'verify-container', checks, - blocker: { code: 'container-readiness', message: komodo.reason, automatic: true } }); continue; } - const status = await probeServerStatus(job.server, job.submitted_at); - if (!stillCheckingCreation(id)) continue; - checks.push({ key: 'game', label: 'Game status', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'waiting', - checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason }); - if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason, { phase: 'verify-game', checks, - blocker: { code: 'game-readiness', message: status.reason, automatic: true } }); continue; } - const actualMap = status.map; - checks.push({ key: 'map', label: 'Selected engine map', source: status.source, result: actualMap === job.expected_map ? 'pass' : 'waiting', - checkedAt: status.observedAt, receivedAt: status.receivedAt, expected: job.expected_map, actual: actualMap }); - if (actualMap !== job.expected_map) { - const message = 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'); - deploymentStage(id, message, { phase: 'verify-map', checks, blocker: { code: 'map-mismatch', message, automatic: true, - href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } }); - continue; - } - verifyingPlutonium = true; - recordProgress(id, { phase: 'verify-plutonium', checks }); - // Refresh only the release, then probe this instance. Fleet-wide checks - // can run independently and must not be required to complete a launch. - await checkRelease(); - if (!stillCheckingCreation(id)) continue; - const revision = await probeVersion(job.server); - if (!stillCheckingCreation(id)) continue; - const target = freshRelease(); - const published = release(); - const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as - { checked_at: string; error: string | null } | undefined; - checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest', - result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target, - message: target === null ? published.error ?? 'The production revision could not be verified.' : null }); - checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon', - result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass', - checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error }); - if (target === null || revision === null) { - const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable') - : 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable'); - deploymentStage(id, message, { phase: 'verify-plutonium', checks, - blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } }); - continue; - } - if (revision < target) { - const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`; - db().transaction(() => { - deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null }); - finishDeployment(id, { ok: false, message }); - })(); - continue; - } - db().transaction(() => { - deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') - + `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, { - phase: 'verify-plutonium', checks, blocker: null, checkpoints: [ - { key: 'container-action', label: 'Container startup verified' }, - { key: 'verify-container', label: 'Container readiness verified' }, - { key: 'verify-game', label: 'Game status verified' }, - { key: 'verify-map', label: 'Selected engine map verified' }, - { key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` } - ] - }); - finishDeployment(id, { ok: true }); - })(); - } catch (err) { - if (!stillCheckingCreation(id)) continue; - const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container'; - const remaining = checks.filter(check => check.key !== key); - remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness', - source: null, result: 'unknown', message: (err as Error).message }); - deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, { - phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining, - blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } }); - } - } - - // Snapshot the budget once. Fast success/failure must not turn a single - // worker pass into an unbounded stream of expensive container startups. - const queue = creationQueue(); - const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length); - const eligible = queue.queued.filter(({ id }) => { - if (nativeDeployment(id)) return false; - const blocker = creationBlocker(id); - if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker }); - else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity', - message: available ? 'Eligible for startup admission in request order.' : 'Waiting for startup capacity to be released.', automatic: true } }); - return !blocker; - }); - for (const { id } of eligible.slice(0, available)) { - if (creating || destroying.size > 0) break; - // API calls can cancel a selected job or resume an older startup while - // a previous provision awaits Komodo. Recheck without expanding this pass. - if (creationQueue().active.length >= fleetSettings().createConcurrency) break; - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue; - const blocker = creationBlocker(id); - if (blocker) { - deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker }); - continue; - } - try { outcomes.set(id, await provisionCreation(id)); } - catch (err) { - const failure = err instanceof Error ? err : new Error(String(err)); - outcomes.set(id, failure); - console.error('[creation] provisioning failed:', failure); - } - } - return outcomes; -} diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 5426d8b..f40437c 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -45,7 +45,7 @@ export function nativeLaunchSelection(input: Record, game: stri if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) { throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`); } - const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; + const image = input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image; if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); const automatic = input.automatic === true || !input.hostId; if (!automatic) { @@ -673,8 +673,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const recipe = prepared.recipe; db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?') .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id); - if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) - VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), + if (job.action === 'create') db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) + VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port); for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create'); recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } }); diff --git a/src/lib/server/provisioning/progress.ts b/src/lib/server/provisioning/progress.ts index 125219d..c476207 100644 --- a/src/lib/server/provisioning/progress.ts +++ b/src/lib/server/provisioning/progress.ts @@ -37,7 +37,7 @@ export function readProgress(row: { observation_json?: string | null }): Provisi || (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null; if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key)) && typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result)) - && (c.source === null || ['komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source))) + && (c.source === null || ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source))) && typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null; if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string' && typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null; diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 1a568fc..4dd4a47 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -1,10 +1,8 @@ import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; -import { komodoConfigured, rconConfigured } from '../config.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; import { queueRebuildServer } from './instances.js'; -import { restartRequestInFlight } from './restarts.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; import { recordProgress } from './progress.js'; import { nativeInstance, unplacedNativeCreation } from '../backends/native.js'; @@ -51,22 +49,6 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id; if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id); } - // Only not-yet-submitted campaign changes can share an activation. A later - // file deployment must never be credited to an already running restart. - // Restorations need their own activation. An older campaign can have - // stale file preconditions after a beta; its failure must not strand - // the rollback that would restore the server. - const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id); - if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) { - id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' - AND d.state = 'waiting' AND d.submitted_at IS NULL - AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL) - AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') - AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') - AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id - WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback') - ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id; - } if (!id) { id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); @@ -138,7 +120,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; - if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' }; + if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' }; return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; if (blocker) return { ok: false as const, status: 409, error: blocker }; @@ -187,7 +169,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract(); -export const restartRequestInFlight = (server: string) => requestsInFlight.has(server); interface RestartSlot { server: string; @@ -37,7 +26,7 @@ export function restartRecoveryPending(server: string): boolean { /** The queued restart may still expose the previous healthy container. */ export function restartSettling(server: string): boolean { const slot = slotFor(server); - return requestsInFlight.has(server) || !!(slot && !slot.healthy_at + return !!(slot && !slot.healthy_at && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS); } @@ -46,7 +35,6 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt } export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null { - if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true }; const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined; @@ -67,167 +55,6 @@ export function rolloutBlocker(server: string, operationId = '', action: Runtime ? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null; } -export type RolloutRestartResult = - | { kind: 'started' } - | { kind: 'queued'; message: string } - | { kind: 'failed'; message: string }; - -/** Reserve capacity before any async I/O; callers still check for players first. */ -export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise { - const operationId = options.operationId; - const waitReason = db().transaction(() => { - if (provisioningCancelled(operationId)) return 'Operation was cancelled'; - if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted'; - const reason = rolloutWaitReason(server, operationId, options.action); - if (reason) return reason; - db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) - ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, - checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`) - .run(server, new Date().toISOString(), operationId); - recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } }); - return null; - }).immediate(); - if (waitReason) return { kind: 'queued', message: waitReason }; - - // Every gate, including rollout capacity, has cleared. No async work may - // intervene between applying these files and submitting the restart. - try { - applyStagedFilesForOperation(operationId, server); - } catch (err) { - const message = `File activation failed: ${(err as Error).message}`; - db().transaction(() => { - db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); - recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } }); - finishDeployment(operationId, { ok: false, message }); - })(); - return { kind: 'failed', message }; - } - db().transaction(() => { - db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId); - deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', { - phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, - command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` } - }); - })(); - requestsInFlight.set(server, operationId); - db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL, - game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); - try { - const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server); - if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' }; - if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; - if (result.ok) { - db().transaction(() => { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId); - deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', { - phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' }, - checkpoint: { key: 'container-action', label: 'Container command accepted' }, - blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true } - }); - })(); - return { kind: 'started' }; - } - // These failures cannot have queued a remote restart. Other failures - // (especially a timeout) may have: retain capacity until recovery. - db().transaction(() => { - if (result.kind === 'no-name' || result.kind === 'unconfigured') { - db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); - recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message }, - recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } }); - } else { - db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId); - recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } }); - } - finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` }); - })(); - return { kind: 'failed', message: `komodo: ${result.message}` }; - } catch (err) { - if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' }; - if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; - const message = `komodo: ${(err as Error).message}`; - db().transaction(() => { - db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId); - recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } }); - finishDeployment(operationId!, { ok: false, message }); - })(); - return { kind: 'failed', message }; - } finally { - // Start the grace period at completion, including slow/failed requests. - db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?') - .run(new Date().toISOString(), server, operationId); - if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server); - } -} - -let refreshInFlight: Promise | null = null; - -/** Poll all occupied slots, even if their originating beta was already closed. */ -export function refreshRestartRollout(): Promise { - if (refreshInFlight) return refreshInFlight; - refreshInFlight = refresh().finally(() => { refreshInFlight = null; }); - return refreshInFlight; -} - -async function refresh(): Promise { - const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server') - .all() as RestartSlot[]; - for (const slot of slots) { - if (slot.operation_id && nativeDeployment(slot.operation_id)) continue; - if (requestsInFlight.has(slot.server)) continue; - // Recovery capacity transferred to a rebuild remains reserved until - // teardown. Its health is verified by the creation worker, never here. - if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND ( - (d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health')) - OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS ( - SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`) - .get(slot.operation_id)) continue; - if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue; - let error: string | null = null; - let players: number | null = null; - const checks: NonNullable = []; - try { - const health = await komodoInspectHealthy(slot.server); - checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null, - checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason }); - if (!health.ok) { - error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`; - } else { - const status = await probeServerStatus(slot.server, slot.requested_at); - checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null, - checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, - message: status.ok ? null : status.reason }); - if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`; - else players = status.players; - } - } catch (err) { - error = `health check: ${(err as Error).message}`; - checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error }); - } - const now = new Date().toISOString(); - db().transaction(() => { - const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ? - WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`) - .run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id); - if (!checked.changes) return; - if (slot.operation_id) recordProgress(slot.operation_id, { - phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now, - blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null, - recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id, - message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' }, - checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }] - }); - if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) { - db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id); - if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error); - else { - deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)'); - finishDeployment(slot.operation_id, { ok: true }); - } - } - })(); - } -} - export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { const slot = slotFor(server); if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index bc24695..370a02b 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -1,24 +1,11 @@ import { maintenanceAllowed } from '../maintenance/policies.js'; -import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; import { db } from '../db/index.js'; -import { provisioningCancelled } from './cancellation.js'; -import { komodoConfigured, rconConfigured } from '../config.js'; import { getServer } from '../fleet/servers.js'; -import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js'; -import { processCreations, processDestructions, rebuildServer } from './instances.js'; -import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js'; -import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js'; -import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js'; +import { rolloutBlocker } from './restarts.js'; import { cancelProvisioning } from './requests.js'; import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; -import { recordProgress } from './progress.js'; -import { nativeInstance, nativeDeployment } from '../backends/native.js'; import { processNativeProvisioning } from './native.js'; -import { trackActivity } from '../pipeline/activity.js'; - -const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']); -export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? ''); interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string } const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[]; @@ -92,20 +79,13 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin } interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string } -let ticking: Promise | null = null; let awaitedPass: Promise | null = null; export function processProvisioning(options: { wait?: boolean } = {}): Promise { if (options.wait !== false && awaitedPass) return awaitedPass; - // Native jobs own their individual flights. A slow observation must not - // prevent the next scheduler pass admitting newly arrived native work. - const native = nativePass(options); - if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run) - .finally(() => { ticking = null; }); - if (options.wait === false) { - void ticking.catch(error => console.error('[worker] legacy provisioning error:', error)); - return native; - } - awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; }); + // Each job owns its flight, so slow observations do not block new work. + const pass = nativePass(options); + if (options.wait === false) return pass; + awaitedPass = pass.finally(() => { awaitedPass = null; }); return awaitedPass; } @@ -126,148 +106,3 @@ function nativePass(options: { wait?: boolean }): Promise { return blocker(operation.server, operation.id, typed.action)?.message ?? null; }, options); } - -async function run(): Promise { - await processCreations(); - await processDestructions(); - await refreshRestartRollout(); - const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments - WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health')) - ORDER BY started_at, rowid`).all() as Operation[]; - for (const operation of operations) { - if (nativeDeployment(operation.id)) continue; - try { await processOperation(operation); } - catch (err) { - if (provisioningCancelled(operation.id)) continue; - // Observation failures are retried; remote mutations are never replayed. - deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, { - checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } - }); - } - } -} - -async function processOperation(operation: Operation): Promise { - const { id, server, action } = operation; - if (provisioningCancelled(id)) return; - // An explicit cleanup resume follows confirmed remote deletion. Complete - // that durable checkpoint even if the incident now needs attention or the - // Watchdog is paused; never strand the reserved replacement identity. - if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; } - if (operation.state === 'waiting') { - const heldBy: Source[] = []; - const eligibility = sourceEligibility(operation, source => heldBy.push(source)); - if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; } - if (eligibility === 'held') { - const source = heldBy[0]; - const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}` - : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}` - : source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}` - : `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null; - deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [], - blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } }); - return; - } - } - if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; } - const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog'); - if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', { - phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' } - }); return; } - if (operation.state === 'awaiting-health') { - const health = await komodoInspectHealthy(server); - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return; - recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo', - result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null, - message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null - : { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } }); - if (isStopped(health.state)) { - db().transaction(() => { - db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server); - deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } }); - finishDeployment(id, { ok: true }); - })(); - } else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : '')); - return; - } - const reason = blocker(server, id, action); - if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; } - const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s - JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id - WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id); - const epoch = latestGameSubmission(server); - const health = await komodoInspectHealthy(server); - if (provisioningCancelled(id)) return; - if (healthRecovery && epoch !== latestGameSubmission(server)) return; - if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) { - deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [], - blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } }); - return; - } - if (action !== 'start' && !isStopped(health.state)) { - const evidence = await probePlayerEvidence(server); - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; - const players = evidence.players; - if (healthRecovery && epoch !== latestGameSubmission(server)) return; - if (healthRecovery && health.ok && players !== null) { - // The fresh occupancy check also proves that the game recovered. - // Empty is not a reason to execute an obsolete health restart. - resolveRecoveredHealthOperation(id, players, evidence.source); - return; - } - recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', - source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }], - blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present', - message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } }); - db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id); - if (players !== 0) { - deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`); - return; - } - } else if (action !== 'start') { - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; - recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption', - source: 'komodo', result: 'pass', actual: health.state ?? null }] }); - } - // Requests can arrive during the probes. Recheck before claiming the action. - const latestReason = blocker(server, id, action); - if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; } - if (sourceEligibility(operation) !== 'ready') return; - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; - if (action === 'rebuild') { await rebuildServer(id); return; } - const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id); - if (!claim.changes) return; - if (action === 'start' || action === 'restart') { - const result = await restartInRollout(server, { operationId: id, action }); - if (provisioningCancelled(id)) return; - if (result.kind === 'queued') { - db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); - deploymentStage(id, result.message); - } - return; - } - db().transaction(() => { - db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id); - deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [], - checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } }); - })(); - // Persist intent first; a lost reply must not issue another stop later. - try { - const result = await komodoRuntimeAction(server, 'stop'); - if (provisioningCancelled(id)) return; - if (!result.ok) { - recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } }); - finishDeployment(id, { ok: false, message: result.message }); return; - } - db().transaction(() => { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); - deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped', - checkpoint: { key: 'container-action', label: 'Stop command accepted' }, - command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } }); - })(); - } catch (err) { - if (provisioningCancelled(id)) return; - recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } }); - finishDeployment(id, { ok: false, message: (err as Error).message }); - } -} diff --git a/src/lib/ui/components/ProvisioningProgress.svelte b/src/lib/ui/components/ProvisioningProgress.svelte index 8086f3f..5751498 100644 --- a/src/lib/ui/components/ProvisioningProgress.svelte +++ b/src/lib/ui/components/ProvisioningProgress.svelte @@ -9,7 +9,7 @@ const progress = $derived(detail.progress); let expanded = $state>({}); const evidenceLabels = { pass: 'Passed', waiting: 'Waiting', fail: 'Failed', unknown: 'Unknown' }; - const sources = { komodo: 'Komodo', agent: 'Host agent', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' }; + const sources = { agent: 'Host agent', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' };
diff --git a/src/lib/ui/komodoError.ts b/src/lib/ui/komodoError.ts deleted file mode 100644 index e6f2b02..0000000 --- a/src/lib/ui/komodoError.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type KomodoPortFailure = - | 'unconfigured' - | 'no-name' - | 'unreachable' - | 'http-error' - | 'unparseable' - | 'no-udp-in-response'; - -/** Short fix hint per failure reason, shown under the Komodo probe result. */ -export function komodoFixHint(reason: KomodoPortFailure, httpStatus?: number): string { - switch (reason) { - case 'unconfigured': - return 'Set KOMODO_ADDRESS, KOMODO_API_KEY, and KOMODO_API_SECRET in the container env.'; - case 'no-name': - return 'This server row has no komodo_name — set it via SQL or via the fleet config.'; - case 'unreachable': - return 'Network, DNS, or timeout. Check that KOMODO_ADDRESS is reachable from this container.'; - case 'http-error': - if (httpStatus === 401 || httpStatus === 403) return 'Auth rejected — verify KOMODO_API_KEY and KOMODO_API_SECRET.'; - if (httpStatus === 404) return 'Endpoint or deployment not found — check komodo_name matches the Komodo UI exactly.'; - return `Komodo returned ${httpStatus ?? 'an error'}. Check the endpoint path and credentials.`; - case 'unparseable': - return 'Response was not JSON — is KOMODO_ADDRESS pointing at the Core API (not the Periphery)?'; - case 'no-udp-in-response': - return 'Parsed the response but no UDP port matched our known shapes. See the snippet below and extend extractUdpPort.'; - } -} diff --git a/src/lib/ui/provisioning-presentation.ts b/src/lib/ui/provisioning-presentation.ts index 7683778..5c31fe8 100644 --- a/src/lib/ui/provisioning-presentation.ts +++ b/src/lib/ui/provisioning-presentation.ts @@ -8,35 +8,27 @@ export interface ProgressGroup { key: string; label: string; steps: ProgressStep const phaseLabels: Record = { requested: 'Request recorded', eligibility: 'Check eligibility and capacity', 'startup-queue': 'Wait for a startup slot', - prepare: 'Prepare server files', 'create-deployment': 'Create Komodo deployment', 'container-action': 'Start container', - 'remove-container': 'Stop and remove container', 'delete-deployment': 'Delete Komodo deployment', cleanup: 'Remove files and registration', + prepare: 'Prepare server files', 'container-action': 'Start container', + cleanup: 'Remove files and registration', 'verify-container': 'Confirm container readiness', 'verify-game': 'Verify game status', 'verify-map': 'Verify selected engine map', 'verify-plutonium': 'Verify Plutonium revision', 'verify-stopped': 'Confirm container is stopped' }; type GroupDefinition = { key: string; label: string; phases: ProvisioningPhase[] }; const request: GroupDefinition = { key: 'request', label: 'Request and eligibility', phases: ['requested', 'eligibility'] }; -const creation: GroupDefinition = { key: 'create', label: 'Create and start', phases: ['startup-queue', 'prepare', 'create-deployment', 'container-action'] }; -const removal: GroupDefinition = { key: 'remove', label: 'Remove the instance', phases: ['remove-container', 'delete-deployment', 'cleanup'] }; const readiness: GroupDefinition = { key: 'verify', label: 'Verify readiness', phases: ['verify-container', 'verify-game', 'verify-map', 'verify-plutonium'] }; -function definitions(action: ProvisioningAction, native = false): GroupDefinition[] { - if (native) { - const command: GroupDefinition = { key: 'command', label: `${actionLabels[action]} on host agent`, phases: ['container-action'] }; - if (action === 'create' || action === 'rebuild') return [request, - { key: 'create', label: action === 'rebuild' ? 'Rebuild on host agent' : 'Create and start on host agent', phases: ['startup-queue', 'prepare', 'container-action'] }, readiness]; - if (action === 'destroy') return [request, command, { key: 'cleanup', label: 'Retire instance', phases: ['cleanup'] }]; - } - if (action === 'create') return [{ ...request, phases: ['requested'] }, creation, readiness]; - if (action === 'rebuild') return [request, removal, { ...creation, label: 'Create a fresh instance' }, readiness]; - if (action === 'destroy') return [request, removal]; - return [request, { key: 'command', label: `${actionLabels[action]} the existing container`, phases: ['container-action'] }, - { ...readiness, phases: action === 'stop' ? ['verify-stopped'] : ['verify-container', 'verify-game'] }]; +function definitions(action: ProvisioningAction): GroupDefinition[] { + const command: GroupDefinition = { key: 'command', label: `${actionLabels[action]} on host agent`, phases: ['container-action'] }; + if (action === 'create' || action === 'rebuild') return [request, + { key: 'create', label: action === 'rebuild' ? 'Rebuild on host agent' : 'Create and start on host agent', phases: ['startup-queue', 'prepare', 'container-action'] }, readiness]; + if (action === 'destroy') return [request, command, { key: 'cleanup', label: 'Retire instance', phases: ['cleanup'] }]; + return [request, command, { ...readiness, phases: action === 'stop' ? ['verify-stopped'] : ['verify-container', 'verify-game'] }]; } export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[] { const progress = detail.progress; if (!progress) return []; - const groups = definitions(detail.deployment.action, detail.backend === 'native'); + const groups = definitions(detail.deployment.action); const phases = groups.flatMap(group => group.phases); const current = progress.phase ? phases.indexOf(progress.phase) : -1; const active = activeProvisioningStates.includes(detail.deployment.state); @@ -54,11 +46,10 @@ export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[] : isCurrent && active ? progress.blocker || ['waiting', 'queued'].includes(detail.deployment.state) ? 'waiting' : 'current' : current >= 0 && index >= current && detail.deployment.state === 'cancelled' ? 'cancelled' : active && current >= 0 && index > current ? 'pending' : 'unknown'; - const label = detail.backend === 'native' && key === 'startup-queue' ? 'Admit to startup capacity' - : detail.backend === 'native' && key === 'prepare' ? 'Prepare launch payload' - : detail.backend === 'native' && key === 'container-action' ? `${actionLabels[detail.deployment.action]} instance through host agent` - : key === 'container-action' && ['start', 'restart', 'stop'].includes(detail.deployment.action) - ? `${actionLabels[detail.deployment.action]} existing container` : phaseLabels[key]; + const label = key === 'startup-queue' ? 'Admit to startup capacity' + : key === 'prepare' ? 'Prepare launch payload' + : key === 'container-action' ? `${actionLabels[detail.deployment.action]} instance through host agent` + : phaseLabels[key]; return { key, label, state, at: check?.checkedAt ?? checkpoint?.at ?? (isCurrent ? progress.phaseStartedAt : null) }; }); const states = steps.map(step => step.state); @@ -168,7 +159,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation) && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage) && nullableText(value.cancellation.error))) return false; - if (value.backend !== undefined && !['native', 'komodo'].includes(value.backend as string)) return false; + if (value.backend !== undefined && !['native'].includes(value.backend as string)) return false; if (value.workerActivity !== undefined && (!Array.isArray(value.workerActivity) || !value.workerActivity.every(item => record(item) && text(item.key) && text(item.label) && timestamp(item.startedAt)))) return false; if (value.agentOperation !== undefined && value.agentOperation !== null && !(record(value.agentOperation) @@ -212,7 +203,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi if (!Array.isArray(progress.checks) || !progress.checks.every(check => record(check) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(check.key as string) && ['pass', 'waiting', 'fail', 'unknown'].includes(check.result as string) - && [null, 'komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(check.source as null | string) + && [null, 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(check.source as null | string) && text(check.label) && timestamp(check.checkedAt) && nullableTime(check.receivedAt) && scalar(check.expected) && scalar(check.actual) && nullableText(check.message))) return false; if (!Array.isArray(progress.checkpoints) || !progress.checkpoints.every(checkpoint => record(checkpoint) diff --git a/src/lib/ui/rconError.ts b/src/lib/ui/rconError.ts index 4e7180b..6f24d7a 100644 --- a/src/lib/ui/rconError.ts +++ b/src/lib/ui/rconError.ts @@ -30,7 +30,7 @@ export function rconFixHint(reason: RconFailure): string { case 'no-config': return 'Set RCON_PASSWORD in the container env and restart.'; case 'no-port': - return 'Set game_port on the server row, or configure Komodo so it can resolve the port.'; + return 'Check the host agent and the game port assigned to this server.'; case 'send-error': return 'DNS or route problem — check the RCON host is reachable from this container.'; case 'timeout': diff --git a/src/lib/ui/stores/fleet.ts b/src/lib/ui/stores/fleet.ts index 7be486c..45437be 100644 --- a/src/lib/ui/stores/fleet.ts +++ b/src/lib/ui/stores/fleet.ts @@ -13,7 +13,7 @@ export function createFleetStore(serverName?: string) { }); const fleetError = writable(''); const fleetRefreshing = writable(false); - const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null); + const fleetIntegrations = writable<{ rcon: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null); const observed = new Map(); const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => { @@ -63,7 +63,6 @@ export function createFleetStore(serverName?: string) { return next; }); - let snapshotServerNow = 0; function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable>; serverNow: number }) { if (!browser || snapshot.serverNow < snapshotServerNow) return; @@ -74,8 +73,6 @@ export function createFleetStore(serverName?: string) { } function getIntegrations() { return get(fleetIntegrations); } - - let inFlight: Promise | null = null; let refreshRequested = false; function refreshFleet(afterCurrent = false): Promise { diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index d55b596..8f89772 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -165,8 +165,6 @@ {/if} {#if integrations?.native && !integrations.hostsReady}
Prepare a game host · {integrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}Review hosts →
- {:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !integrations.native && (!integrations.komodo || !integrations.rcon)} -
Finish system setup · {!integrations.komodo ? 'Komodo' : ''}{!integrations.komodo && !integrations.rcon ? ' and ' : ''}{!integrations.rcon ? 'RCON' : ''} not configured.Review setup →
{/if} {#if problems.length || attentionOperations.length || starting || provisioning} diff --git a/src/routes/api/hosts/+server.ts b/src/routes/api/hosts/+server.ts index f42396f..702c222 100644 --- a/src/routes/api/hosts/+server.ts +++ b/src/routes/api/hosts/+server.ts @@ -4,7 +4,7 @@ import { hostApiError, installerCommand, objectBody, requireManagementOrigin } f import { presentHostPreparation } from '$lib/server/hosts/preparation.js'; import { config } from '$lib/server/config.js'; -export function GET({ request }: { request: Request }) { requireManagementOrigin(request); return json({ hosts: listHosts().map(presentHostPreparation), defaultRuntimeImage: config.komodo.defaultImage }); } +export function GET({ request }: { request: Request }) { requireManagementOrigin(request); return json({ hosts: listHosts().map(presentHostPreparation), defaultRuntimeImage: config.defaultRuntimeImage }); } export async function POST({ request }: { request: Request }) { requireManagementOrigin(request); try { diff --git a/src/routes/api/hosts/[id]/+server.ts b/src/routes/api/hosts/[id]/+server.ts index b578acb..2cebbb8 100644 --- a/src/routes/api/hosts/[id]/+server.ts +++ b/src/routes/api/hosts/[id]/+server.ts @@ -10,7 +10,7 @@ import { kickProvisioning } from '$lib/server/pipeline/worker.js'; type Event = { request: Request; params: { id: string } }; export function GET({ request, params }: Event) { requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.'); - return json({ host: presentHostPreparation(host), instances: hostInstances(host.id), serverNow: Date.now(), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id) }); + return json({ host: presentHostPreparation(host), instances: hostInstances(host.id), serverNow: Date.now(), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.defaultRuntimeImage, deletion: hostDeletionStatus(host.id) }); } export async function PATCH({ request, params }: Event) { requireManagementOrigin(request); diff --git a/src/routes/api/servers/[name]/diagnose-network/+server.ts b/src/routes/api/servers/[name]/diagnose-network/+server.ts deleted file mode 100644 index c52f488..0000000 --- a/src/routes/api/servers/[name]/diagnose-network/+server.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { json } from '@sveltejs/kit'; -import type { RequestHandler } from './$types'; -import { getServer } from '$lib/server/fleet/servers.js'; -import { config } from '$lib/server/config.js'; -import { tcpProbeAll } from '$lib/server/integrations/netProbe.js'; - -/** - * Multi-target TCP reachability probe. Fires parallel connect attempts at a - * fixed matrix of candidate hosts + the configured KOMODO_ADDRESS host, so - * we can see at a glance which network paths reach anything and where - * packets die (DNS, routing, or refused-at-port). - */ -export const POST: RequestHandler = async ({ params }) => { - const name = params.name; - if (!name) return json({ ok: false, error: 'missing server name' }, { status: 400 }); - - const row = getServer(name); - if (!row) return json({ ok: false, error: `unknown server: ${name}` }, { status: 404 }); - - // Parse the configured Komodo URL to extract host+port. - let komodoHost: string | null = null; - let komodoPort: number | null = null; - if (config.komodo.address) { - try { - const u = new URL(config.komodo.address); - komodoHost = u.hostname; - komodoPort = u.port ? Number.parseInt(u.port, 10) : u.protocol === 'https:' ? 443 : 80; - } catch { - /* leave null */ - } - } - - const probePort = komodoPort ?? 9120; - - // Build the trial matrix. Every trial hits the same target port (the - // Komodo port) via a different network path, so we can see which paths - // reach the same service. - const trials: Array<{ label: string; host: string; port: number }> = [ - { label: `127.0.0.1:${probePort} (container loopback)`, host: '127.0.0.1', port: probePort }, - { label: `host.docker.internal:${probePort} (bridge gateway alias)`, host: 'host.docker.internal', port: probePort }, - { label: `172.17.0.1:${probePort} (default bridge gateway)`, host: '172.17.0.1', port: probePort } - ]; - - if (komodoHost && !trials.some((t) => t.host === komodoHost)) { - trials.push({ - label: `${komodoHost}:${probePort} (KOMODO_ADDRESS)`, - host: komodoHost, - port: probePort - }); - } - - // RCON's default host is worth testing too — even though RCON is UDP, - // if TCP to the same host also fails, it's a routing/firewall issue, - // not a UDP-specific one. - const rconHost = row.rcon_host ?? config.rcon.host; - if (rconHost && !trials.some((t) => t.host === rconHost)) { - trials.push({ - label: `${rconHost}:${probePort} (RCON_HOST)`, - host: rconHost, - port: probePort - }); - } - - const results = await tcpProbeAll(trials, config.httpTimeoutMs); - - return json({ - ok: true, - config: { - komodoAddress: config.komodo.address || null, - komodoHost, - komodoPort, - rconHost, - probeTimeoutMs: config.httpTimeoutMs - }, - results - }); -}; diff --git a/src/routes/api/servers/[name]/probe-komodo/+server.ts b/src/routes/api/servers/[name]/probe-komodo/+server.ts deleted file mode 100644 index c8a17ac..0000000 --- a/src/routes/api/servers/[name]/probe-komodo/+server.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { json } from '@sveltejs/kit'; -import type { RequestHandler } from './$types'; -import { komodoResolvePortDetailed } from '$lib/server/integrations/komodo.js'; -import { getServer, setServerField } from '$lib/server/fleet/servers.js'; - -/** - * Trigger a fresh Komodo GetDeployment lookup for a specific server and - * return the structured result — including a snippet of the response so - * the UI can render "here's where we found the port" (or "here's what - * Komodo returned; extend extractUdpPort for this shape"). - * - * On success, the resolved port is written to servers.game_port so the - * next RCON attempt uses it without waiting for another sweep tick. - */ -export const POST: RequestHandler = async ({ params }) => { - const name = params.name; - if (!name) return json({ ok: false, reason: 'no-name', detail: 'missing server name' }, { status: 400 }); - - const row = getServer(name); - if (!row) return json({ ok: false, reason: 'no-name', detail: `unknown server: ${name}` }, { status: 404 }); - - const result = await komodoResolvePortDetailed(name); - if (result.ok) { - setServerField(name, 'game_port', result.port); - } - return json(result); -}; diff --git a/src/routes/api/system-status/+server.ts b/src/routes/api/system-status/+server.ts index f5d5a10..bc014d5 100644 --- a/src/routes/api/system-status/+server.ts +++ b/src/routes/api/system-status/+server.ts @@ -1,59 +1,30 @@ import { json } from '@sveltejs/kit'; -import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js'; import { listServers } from '$lib/server/fleet/servers.js'; import { listServerKeys } from '$lib/server/fleet/server-keys.js'; import { serverObservation } from '$lib/server/fleet/status.js'; import { freshStatusTimestamp } from '$lib/server-freshness.js'; -import { tcpProbe } from '$lib/server/integrations/netProbe.js'; import type { SetupCheck, SystemStatus } from '$lib/system-status.js'; import { listHosts } from '$lib/server/hosts/index.js'; import { listLibraries } from '$lib/server/game-files/index.js'; -export async function GET({ url }: { url: URL }) { +export async function GET() { const servers = listServers(); const readyServers = servers.filter(server => serverObservation(server).health === 'online').length; const rconServers = servers.filter(server => !server.last_error && freshStatusTimestamp(server.last_seen_at, Date.now()) && server.container_state === 'running').length; const savedKeys = listServerKeys().length; const hosts = listHosts(); - if (hosts.length || !komodoConfigured()) { - const readyHosts = hosts.filter(host => host.readiness === 'ready'); - const connected = hosts.filter(host => host.connected); - const versions = listLibraries().flatMap(game => game.versions.filter(version => version.id === game.latestVersionId)); - const cached = versions.filter(version => version.caches.some(cache => cache.status === 'ready' && readyHosts.some(host => host.id === cache.hostId))); - const runtimes = connected.filter(host => host.inventory.runtimeImages?.length); - const checks: SetupCheck[] = [ - { id: 'hosts', title: 'Host agents', state: readyHosts.length ? 'ready' : hosts.length ? 'attention' : 'unconfigured', detail: `${connected.length} connected, ${readyHosts.length} deployment-ready, ${hosts.length} enrolled or awaiting enrollment.`, href: '/hosts', action: hosts.length ? 'Review hosts' : 'Add host' }, - { id: 'runtime', title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length ? `${runtimes.length} connected hosts report cached runtime images.` : 'Hosts prepare the configured Plutainer runtime automatically.', href: '/hosts', action: 'View host setup' }, - { id: 'game-files', title: 'T5 / T6 game files', state: cached.length ? 'ready' : 'attention', detail: `${versions.length} games have uploaded files; ${cached.length} have their latest files verified on a ready host.`, href: '/game-files', action: 'Manage game files' }, - { id: 'rcon', title: 'Agent-local game connection', state: rconServers ? 'ready' : 'unverified', detail: rconServers ? `${rconServers} servers have fresh RCON responses.` : 'The host agent checks RCON locally after deployment. Controller access to game ports is not required.', href: servers.length ? '/' : '/hosts', action: servers.length ? 'Review fleet' : 'Review hosts' }, - { id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Validity is checked with Plutonium at startup.`, href: '/server-keys', action: 'Manage keys' }, - { id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Registered servers are awaiting fresh healthy reports. Inspect their host and diagnostics.' : 'Choose a host to launch your first server with the latest game files.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' } - ]; - return json({ checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys } satisfies SystemStatus, { headers: { 'cache-control': 'no-store' } }); - } - const missing = [!config.komodo.address && 'KOMODO_ADDRESS', !config.komodo.apiKey && 'KOMODO_API_KEY', !config.komodo.apiSecret && 'KOMODO_API_SECRET'].filter(Boolean); - const connection: SetupCheck = { id: 'komodo', title: 'Komodo connection', state: komodoConfigured() ? 'unverified' : 'unconfigured', detail: komodoConfigured() ? 'Credentials configured. Run connection check to verify host reachability.' : `Set ${missing.join(', ')} in the app environment.`, href: '#setup-guide', action: 'Setup instructions' }; - if (komodoConfigured() && url.searchParams.get('probe') === '1') { - try { - const address = new URL(config.komodo.address); - if (!['http:', 'https:'].includes(address.protocol)) throw new Error('invalid address'); - const result = await Promise.race([ - tcpProbe('Komodo', address.hostname, Number(address.port || (address.protocol === 'https:' ? 443 : 80)), 3000), - new Promise(resolve => setTimeout(() => resolve(null), 4000)) - ]); - const reachable = result?.outcome === 'connected'; - connection.state = reachable ? 'ready' : 'attention'; - connection.detail = reachable ? 'Host reachable. This network check does not validate API credentials; deployment previews check launch prerequisites.' : 'Host unreachable from Electric Cherry. Check the address, DNS, port, and network access.'; - } catch { connection.state = 'attention'; connection.detail = 'Komodo address is invalid or could not be reached. Check KOMODO_ADDRESS.'; } - } + const readyHosts = hosts.filter(host => host.readiness === 'ready'); + const connected = hosts.filter(host => host.connected); + const versions = listLibraries().flatMap(game => game.versions.filter(version => version.id === game.latestVersionId)); + const cached = versions.filter(version => version.caches.some(cache => cache.status === 'ready' && readyHosts.some(host => host.id === cache.hostId))); + const runtimes = connected.filter(host => host.inventory.runtimeImages?.length); const checks: SetupCheck[] = [ - connection, - { id: 'target', title: 'Deployment host', state: config.komodo.targetServer ? 'ready' : 'unconfigured', detail: config.komodo.targetServer ? `New servers target ${config.komodo.targetServer}.` : 'Set KOMODO_TARGET_SERVER to the exact host name shown in Komodo.', href: '#setup-guide', action: 'Setup instructions' }, - { id: 'rcon', title: 'Game server connection', state: !rconConfigured() ? 'unconfigured' : rconServers ? 'ready' : 'unverified', detail: !rconConfigured() ? 'Set RCON_PASSWORD to the password used by your game servers.' : rconServers ? `${rconServers} servers have fresh successful RCON reports.` : 'RCON password configured. A running server is needed to verify a response.', href: servers.length ? '/' : '#setup-guide', action: servers.length ? 'Review fleet' : 'Setup instructions' }, - { id: 'game-files', title: 'Game files on the host', state: 'unverified', detail: 'T5 and T6 game-file volumes must be populated on the Komodo host. Their contents cannot be verified from this app.', href: '#game-files', action: 'Game-file instructions' }, - { id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Key presence does not confirm validity with Plutonium.`, href: '/server-keys', action: savedKeys ? 'Manage keys' : 'Add a key' }, - { id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' } + { id: 'hosts', title: 'Host agents', state: readyHosts.length ? 'ready' : hosts.length ? 'attention' : 'unconfigured', detail: `${connected.length} connected, ${readyHosts.length} deployment-ready, ${hosts.length} enrolled or awaiting enrollment.`, href: '/hosts', action: hosts.length ? 'Review hosts' : 'Add host' }, + { id: 'runtime', title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length ? `${runtimes.length} connected hosts report cached runtime images.` : 'Hosts prepare the configured Plutainer runtime automatically.', href: '/hosts', action: 'View host setup' }, + { id: 'game-files', title: 'T5 / T6 game files', state: cached.length ? 'ready' : 'attention', detail: `${versions.length} games have uploaded files; ${cached.length} have their latest files verified on a ready host.`, href: '/game-files', action: 'Manage game files' }, + { id: 'rcon', title: 'Agent-local game connection', state: rconServers ? 'ready' : 'unverified', detail: rconServers ? `${rconServers} servers have fresh RCON responses.` : 'The host agent checks RCON locally after deployment. Controller access to game ports is not required.', href: servers.length ? '/' : '/hosts', action: servers.length ? 'Review fleet' : 'Review hosts' }, + { id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Validity is checked with Plutonium at startup.`, href: '/server-keys', action: 'Manage keys' }, + { id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Registered servers are awaiting fresh healthy reports. Inspect their host and diagnostics.' : 'Choose a host to launch your first server with the latest game files.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' } ]; - const body: SystemStatus = { checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys }; - return json(body, { headers: { 'cache-control': 'no-store' } }); + return json({ checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys } satisfies SystemStatus, { headers: { 'cache-control': 'no-store' } }); } diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index 6057a8f..8a052b7 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -29,7 +29,6 @@ import ChatPanel from '$ui/components/ChatPanel.svelte'; import { gametypeLabel } from '$ui/gametype.js'; import { decodeRconError, rconFixHint } from '$ui/rconError.js'; - import { komodoFixHint, type KomodoPortFailure } from '$ui/komodoError.js'; import Card from '$ui/components/primitives/Card.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; @@ -41,29 +40,6 @@ import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte'; import EmptyState from '$ui/components/primitives/EmptyState.svelte'; - interface KomodoProbeResult { - ok: boolean; - port?: number; - reason?: KomodoPortFailure; - detail?: string; - httpStatus?: number; - snippet?: string; - url?: string; - } - type NetOutcome = 'connected' | 'refused' | 'timeout' | 'dns-fail' | 'unreachable' | 'error'; - interface NetTrial { - label: string; host: string; port: number; - resolvedIp: string | null; outcome: NetOutcome; elapsedMs: number; detail?: string; - } - interface NetDiagnosis { - ok: boolean; - config?: { - komodoAddress: string | null; komodoHost: string | null; komodoPort: number | null; - rconHost: string; probeTimeoutMs: number; - }; - results?: NetTrial[]; - error?: string; - } interface RconProbeResult { ok: boolean; endpoint?: { host: string; port: number } | null; @@ -82,7 +58,6 @@ native_instance_id?: string | null; host_id?: string | null; instance_generation?: number | null; game_library_id?: string | null; host?: import('$lib/hosts.js').Host | null; network?: { bindAddress: string; localPort: number; publicAddress: string | null; publicPort: number | null } | null; - komodo_type: string; komodo_name: string; komodo_server: string | null; game_port: number | null; rcon_host: string | null; game: string | null; map: string | null; last_players: number | null; last_checked_at: string | null; @@ -98,9 +73,6 @@ relPath: string; kind?: string; scriptVersionId: number | null; sha256: string; size: number; updatedAt: string; source: string | null; } - interface Integrations { - rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string; - } interface DeploymentRow { id: string; action: string; started_at: string; finished_at: string | null; servers: Array<{ server: string; restart_state: string; players: number | null; error: string | null; files_changed: number }>; @@ -108,7 +80,7 @@ let name = $derived($page.params.name ?? ''); const detail = $derived(createFleetStore(name)); - const { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } = $derived(detail); + const { fleet, fleetServers, fleetError, refreshFleet } = $derived(detail); $effect(() => { detail.hydrateFleet(data.serverSnapshot); return detail.startFleetMonitor(); @@ -117,13 +89,8 @@ const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.serverSnapshot.servers); let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null); const native = $derived(!!server?.native_instance_id); - let integrations = $derived($fleetIntegrations ?? data.serverSnapshot.integrations); let history = $state([]); let inventory = $state([]); - let komodoProbe = $state(null); - let probing = $state(false); - let netDiagnosis = $state(null); - let diagnosing = $state(false); let rconProbe = $state(null); let probingRcon = $state(false); let destroyOpen = $state(false); @@ -171,7 +138,7 @@ tabLoading = tab === 'history' || tab === 'inventory'; return pollVisible(signal => loadTab(currentName, tab, signal), 10000); }); - $effect(() => { name; komodoProbe = null; rconProbe = null; netDiagnosis = null; }); + $effect(() => { name; rconProbe = null; }); const fresh = $derived(server ? freshServer(server, $uiNow) : false); let effectivePlayers = $derived(fresh ? server?.last_players ?? null : null); @@ -214,29 +181,6 @@ return `${ep.host}:${ep.port}`; } - async function probeKomodo(): Promise { - if (probing || !name) return; - probing = true; komodoProbe = null; - try { - const res = await fetch(`/api/servers/${encodeURIComponent(name)}/probe-komodo`, { method: 'POST' }); - komodoProbe = (await res.json()) as KomodoProbeResult; - if (komodoProbe.ok && server) { - await refreshFleet(); - } - } catch (err) { - komodoProbe = { ok: false, reason: 'unreachable', detail: (err as Error).message }; - } finally { probing = false; } - } - async function diagnoseNetwork(): Promise { - if (diagnosing || !name) return; - diagnosing = true; netDiagnosis = null; - try { - const res = await fetch(`/api/servers/${encodeURIComponent(name)}/diagnose-network`, { method: 'POST' }); - netDiagnosis = (await res.json()) as NetDiagnosis; - } catch (err) { - netDiagnosis = { ok: false, error: (err as Error).message }; - } finally { diagnosing = false; } - } async function probeRcon(): Promise { if (probingRcon || !name) return; probingRcon = true; rconProbe = null; @@ -337,8 +281,8 @@ {#if $fleetError}{/if} {#if hasDiagnosticIssue}
-
{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}

Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : !native && integrations && !integrations.rcon ? 'RCON is not configured.' : ''}

-
{#if native && server.host_id}{:else if integrations && (!integrations.rcon || !integrations.komodo)}{/if}
+
{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}

Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : ''}

+
{#if native && server.host_id}{:else}{/if}
{/if}
@@ -410,13 +354,6 @@

Current player count: {server.status_source === 'telemetry' ? 'game telemetry' : server.status_source === 'rcon' ? 'RCON' : 'unavailable'}. RCON probes below show that connection's separate results.

{#if native}
Game connectionRCON runs locally through the host agent
Host agent{server.host?.connectionState ?? 'unknown'}
Last heartbeat{server.host?.lastHeartbeat ? ago(server.host.lastHeartbeat, $uiNow) : 'Not received'}
- {:else if integrations} -
-
Endpoint{endpointLabel(effectiveEndpoint)}
-
RCON password{integrations.rcon ? 'configured' : 'missing'}
-
Komodo{integrations.komodo ? 'configured' : 'missing'}
-
Timeout{integrations.rconTimeoutMs} ms
-
{/if} {#if effectiveReason} @@ -445,14 +382,7 @@ {native ? 'Probe RCON through agent' : 'Probe RCON'} {/if} - {#if !native && !effectiveEndpoint && integrations?.komodo} - - {/if} - {#if !native}{:else if server.host_id}{/if} + {#if server.host_id}{/if}
{#if rconProbe?.ok} @@ -466,91 +396,12 @@ {/if} - {#if !native && netDiagnosis} - {#if netDiagnosis.ok && netDiagnosis.results} - -
- - - - - - - - - {#each netDiagnosis.results as r (r.label)} - - - - - - - - {/each} - -
TrialResolvedOutcomeElapsedDetail
{r.label}{r.resolvedIp ?? '—'} - {r.outcome} - {r.elapsedMs} ms{r.detail ?? ''}
-
- {#if netDiagnosis.config} -
- KOMODO_ADDRESS = {netDiagnosis.config.komodoAddress ?? '(unset)'} -  ·  RCON_HOST = {netDiagnosis.config.rconHost} -  ·  probe timeout = {netDiagnosis.config.probeTimeoutMs} ms -
- {/if} - {:else} -
- Error - {netDiagnosis.error ?? 'network diagnostic failed'} -
- {/if} - {/if} - - {#if !native && komodoProbe} - {#if komodoProbe.ok} -
- Resolved - - Port {komodoProbe.port} cached — RCON will retry on the next tick. - -
- {#if komodoProbe.snippet} -
- Show response snippet - 
{komodoProbe.snippet}
-
- {/if} - {:else} -
- {komodoProbe.reason} - - {komodoProbe.detail}{#if komodoProbe.httpStatus} · HTTP {komodoProbe.httpStatus}{/if} - -
- {#if komodoProbe.url} -
Tried{komodoProbe.url}
- {/if} - {#if komodoProbe.reason} -
{komodoFixHint(komodoProbe.reason, komodoProbe.httpStatus)}
- {/if} - {#if komodoProbe.snippet} - 
{komodoProbe.snippet}
- {/if} - {/if} - {/if} -

Configuration

{#if native}
Host{server.host?.name ?? server.host_id}
Instance{server.native_instance_id}
Generation{server.instance_generation}
Game-file version{server.game_library_id}
{#if server.network}
Local UDP binding{server.network.bindAddress}:{server.network.localPort}
Public / NAT endpoint{server.network.publicAddress || server.host?.publicAddress || server.host?.hostname || 'Not configured'}:{server.network.publicPort ?? server.network.localPort}
{/if} - {:else}
Komodo{server.komodo_type} · {server.komodo_name}
RCON override{server.rcon_host ?? '(default)'}
{/if} + {/if}
Game port{server.game_port ?? '(unresolved)'}
Game{server.game ?? '(unset)'}
Map{server.map ?? '(unset)'}
@@ -777,19 +628,6 @@ margin-top: var(--space-4); } - .url-row { - display: flex; - gap: var(--space-2); - margin-top: var(--space-2); - padding: var(--space-2) var(--space-3); - background: var(--surface-sunken); - border-radius: var(--radius-md); - font-family: var(--font-mono); - font-size: var(--text-xs); - } - .url-row .k { color: var(--text-tertiary); } - .url-row code { color: var(--text-primary); word-break: break-all; } - /* Snippet details */ .snippet-details { margin-top: var(--space-3); @@ -818,60 +656,6 @@ word-break: break-all; } - /* Network table */ - .table-scroll { max-width: 100%; overflow-x: auto; } - .table-scroll:focus-visible { box-shadow: var(--focus-ring); } - .net-table { - min-width: 600px; - width: 100%; - margin-top: var(--space-4); - border-collapse: separate; - border-spacing: 0; - background: var(--surface-sunken); - border: 1px solid var(--border-subtle); - border-radius: var(--radius-md); - overflow: hidden; - font-size: var(--text-sm); - } - .net-table thead th { - text-align: left; - padding: var(--space-3) var(--space-4); - font-family: var(--font-body); - font-size: var(--text-xs); - font-weight: var(--weight-semibold); - text-transform: uppercase; - letter-spacing: var(--tracking-label); - color: var(--text-tertiary); - border-bottom: 1px solid var(--border-subtle); - } - .net-table tbody td { - padding: var(--space-3) var(--space-4); - border-bottom: 1px solid var(--border-subtle); - color: var(--text-primary); - vertical-align: top; - } - .net-table tbody tr:last-child td { border-bottom: none; } - .net-table .num { text-align: right; } - .net-table .detail-cell { - color: var(--text-tertiary); - font-size: var(--text-xs); - word-break: break-word; - max-width: 320px; - } - - .net-meta { - margin-top: var(--space-3); - font-family: var(--font-mono); - font-size: var(--text-xs); - color: var(--text-tertiary); - } - .net-meta code { - color: var(--text-primary); - background: var(--surface-sunken); - padding: 2px 6px; - border-radius: var(--radius-sm); - } - /* Inventory */ .inv-list { display: flex; diff --git a/tests/helpers/agent-files.ts b/tests/helpers/agent-files.ts new file mode 100644 index 0000000..a2911ad --- /dev/null +++ b/tests/helpers/agent-files.ts @@ -0,0 +1,23 @@ +import * as fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { vi } from 'vitest'; + +/** Read-only agent transport backed by fixture files. Mutations still fail. */ +export async function mockAgentFiles() { + const backend = await import('../../src/lib/server/backends/native.js'); + const { serverDir } = await import('../../src/lib/server/config.js'); + return vi.spyOn(backend, 'nativeRequest').mockImplementation(async (instance, kind, payload = {}) => { + const path = join(serverDir(instance.server), String(payload.path ?? '.')); + let result: unknown; + if (kind === 'files.read') { + const content = await fs.readFile(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); + result = { contentBase64: content?.toString('base64') ?? null }; + } else if (kind === 'files.stat') { + const stat = await fs.stat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); + result = stat ? { exists: true, size: stat.size, mtime: stat.mtime.toISOString(), isDirectory: stat.isDirectory(), isFile: stat.isFile() } : null; + } else if (kind === 'files.list') { + result = { entries: (await fs.readdir(path, { withFileTypes: true })).map(entry => ({ name: entry.name, isDirectory: entry.isDirectory(), isFile: entry.isFile() })) }; + } else throw new Error(`Unexpected agent command: ${kind}`); + return result as never; + }); +} diff --git a/tests/helpers/native-host.ts b/tests/helpers/native-host.ts new file mode 100644 index 0000000..98497db --- /dev/null +++ b/tests/helpers/native-host.ts @@ -0,0 +1,33 @@ +import { createHash, randomUUID } from 'node:crypto'; + +/** Seed verified host inventory without copying a complete game distribution. */ +export async function prepareNativeHost(servers: string[] = []) { + const { config } = await import('../../src/lib/server/config.js'); + const hosts = await import('../../src/lib/server/hosts/index.js'); + const libraries = await import('../../src/lib/server/game-files/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const host = hosts.createHost({ name: 'Fixture host', portMin: 28960, portMax: 29060 }); + const inventory = { protocolVersion: 1, cpuCount: 64, cpuUsagePercent: 0, + memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3, + dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, + runtimeImages: [config.defaultRuntimeImage] }; + hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); + const versions = new Map(); + for (const game of ['t5', 't6'] as const) { + const version = libraries.createLibraryImport({ hostId: host.id, game }); + const manifest = JSON.stringify([{ path: 'zone/base.ff', size: 4, sha256: createHash('sha256').update('game').digest('hex') }]); + const hash = createHash('sha256').update(manifest).digest('hex'); + db().prepare("UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=4,published_at=? WHERE id=?") + .run(manifest, hash, new Date().toISOString(), version.id); + libraries.updateHostLibraryCaches(host.id, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: hash }]); + versions.set(game, version.id); + } + const heartbeat = () => hosts.heartbeat(host.id, { ...inventory, libraries: [...versions.values()].map(id => ({ id, state: 'ready' as const })) }); + heartbeat(); + for (const [index, server] of servers.entries()) { + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES(?,?,?,?,?,?,?)`).run(randomUUID(), server, host.id, versions.get(server.startsWith('t6_') ? 't6' : 't5'), + config.defaultRuntimeImage, 28960 + index, new Date().toISOString()); + } + return { host, versions, heartbeat }; +} diff --git a/tests/unit/beta-destruction.test.ts b/tests/unit/beta-destruction.test.ts index 0645b55..e748401 100644 --- a/tests/unit/beta-destruction.test.ts +++ b/tests/unit/beta-destruction.test.ts @@ -1,17 +1,9 @@ +import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { access } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(async () => ({ ok: true })), - komodoDeploy: vi.fn(async () => ({ ok: true })), - komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), - komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), - komodoInspectHealthy: vi.fn(), - komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), - komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); @@ -24,19 +16,18 @@ const server = 't5_zombie_theater_1'; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([otherServer, server]); - for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', - KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test', - SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28980' })) vi.stubEnv(key, value); + vi.stubEnv('RCON_PASSWORD', 'test'); const { db } = await import('../../src/lib/server/db/index.js'); for (const [index, name] of [otherServer, server].entries()) { const map = name === server ? 'theater' : 'cosmodrome'; - db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir) - VALUES (?, 'deployment', ?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name)); + db().prepare(`INSERT INTO servers (name, game, map, game_mode, map_slug, game_port, host_dir) + VALUES (?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name)); } + await prepareNativeHost([otherServer, server]); + const { mockAgentFiles } = await import('../helpers/agent-files.js'); + await mockAgentFiles(); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0); - vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); @@ -55,9 +46,7 @@ async function attachBeta(state = 'running', outcome?: string, deleted = false) async function expectServerPreserved() { const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]); expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toEqual({ name: server }); await expect(access(join(fixture.t5, server))).resolves.toBeUndefined(); } @@ -117,86 +106,6 @@ describe('active beta destruction guard', () => { expect(destroyServer(server)).toMatchObject({ ok: true, deploymentId: expect.any(String) }); }); - it('holds a queued destruction until its beta closes, then completes the same request', async () => { - const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const queued = destroyServer(server); - expect(queued.ok).toBe(true); - const id = await attachBeta(); - await processDestructions(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); - expect(rconGetPlayerCount).not.toHaveBeenCalled(); - await expectServerPreserved(); - db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(new Date().toISOString(), id); - await processDestructions(); - expect(komodoDestroyDeployment).toHaveBeenCalledOnce(); - expect(komodoDeleteDeployment).toHaveBeenCalledOnce(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); - }); - - it('rechecks beta ownership after an in-flight empty-server probe', async () => { - const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - let resolveCount!: (players: number) => void; - vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); - const queued = destroyServer(server); - const work = processDestructions(); - await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); - await attachBeta(); - resolveCount(0); - await work; - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); - await expectServerPreserved(); - }); - - it('also holds rebuild destruction when a beta appears during the occupancy probe', async () => { - const { requestLifecycle, processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - saveServerKey(server, 'saved-key'); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); - let resolveCount!: (players: number) => void; - vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); - const queued = requestLifecycle(server, 'rebuild'); - if (!queued.ok) throw new Error(queued.error); - const work = processLifecycle(); - await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); - await attachBeta(); - resolveCount(0); - await work; - expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); - await expectServerPreserved(); - }); - - it('preserves the cleanup checkpoint and files while an active beta blocks a legacy cleanup resume', async () => { - const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { deploymentStage, finishDeployment, resumeCleanup, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const queued = destroyServer(server); - deploymentStage(queued.deploymentId!, REMOVING_SERVER); - finishDeployment(queued.deploymentId!, { ok: false, message: 'Interrupted local cleanup' }); - const id = await attachBeta(); - expect(resumeCleanup(queued.deploymentId!)).toBe(true); - await processDestructions(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER, - error: expect.stringMatching(/beta/i) }); - expect(getDeployment(queued.deploymentId!)?.canResumeCleanup).toBe(true); - await expectServerPreserved(); - db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(id); - expect(resumeCleanup(queued.deploymentId!)).toBe(true); - await processDestructions(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - }); - it.each(['destroy', 'rebuild'] as const)('requires cancelling a pending %s before opening a beta', async action => { const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); @@ -222,7 +131,7 @@ describe('active beta destruction guard', () => { }); it.each((['destroy', 'rebuild'] as const).flatMap(action => ['failed', 'interrupted'].flatMap(state => - ['Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'].map(stage => ({ action, state, stage })) + ['Stopping and removing container', 'Removing server files and registry'].map(stage => ({ action, state, stage })) )))('rejects beta admission after $action is $state at $stage', async ({ action, state, stage }) => { const { beginDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); diff --git a/tests/unit/beta-plans.test.ts b/tests/unit/beta-plans.test.ts index 245dff2..c6e69e9 100644 --- a/tests/unit/beta-plans.test.ts +++ b/tests/unit/beta-plans.test.ts @@ -6,9 +6,26 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; import { defaultPlan, requirement, type BetaPlan } from '../../src/lib/beta-plan.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn() })); +// Planner tests observe the provisioning contract. Host command delivery and +// disruption safety are exercised by the native provisioning suites. +vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ + processProvisioning: async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { applyStagedFilesForOperation } = await import('../../src/lib/server/fleet/staged-files.js'); + const { probePlayerEvidence } = await import('../../src/lib/server/fleet/status-evidence.js'); + const jobs = db().prepare("SELECT id,server,state FROM server_deployments WHERE action='restart' AND state IN ('waiting','awaiting-health')").all() as { id: string; server: string; state: string }[]; + for (const job of jobs) { + if (job.state === 'awaiting-health') { + db().prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id=?").run(new Date().toISOString(), job.id); + } else if ((await probePlayerEvidence(job.server)).players === 0) { + applyStagedFilesForOperation(job.id, job.server); + db().prepare("UPDATE server_deployments SET state='awaiting-health',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); + } + } + } +})); let fixture: Fixture; const names = ['t5_zombie_theater_1', 't5_zombie_theater_2']; const now = () => new Date().toISOString(); @@ -21,14 +38,11 @@ beforeEach(async () => { await writeFile(join(fixture.t5, name, 'logs/console.log'), 'old script error\n'); await writeFile(join(fixture.t5, name, 'logs/games_sp.log'), 'InitGame:\n 0:00 J;old;1;old\nExitLevel: executed\n'); } - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test'); - vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2'); + + vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(komodoRestart).mockResolvedValue({ ok: true }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 1 }, rawReply: '', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); }); @@ -57,14 +71,12 @@ describe('durable beta test plans', () => { it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => { const id = await openPlan(defaultPlan()); const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); await postTelemetry(gameEvent('heartbeat', 4, 2)); await tick(); - expect(komodoRestart).not.toHaveBeenCalled(); expect(listBetaServers(id)[0].stage).toBe('awaiting-empty'); await postTelemetry(gameEvent('heartbeat', 4, 0)); await tick(); - expect(komodoRestart).toHaveBeenCalledTimes(1); + expect(listBetaServers(id)[0].stage).toBe('awaiting-health'); }); it('holds file deployment until each wave passes, and rollback leaves undeployed files alone', async () => { diff --git a/tests/unit/beta-promotion.test.ts b/tests/unit/beta-promotion.test.ts index e10d463..b7a37ee 100644 --- a/tests/unit/beta-promotion.test.ts +++ b/tests/unit/beta-promotion.test.ts @@ -9,7 +9,7 @@ let fixture: Fixture; const servers = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(servers); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', ''); + vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); diff --git a/tests/unit/beta-staging.test.ts b/tests/unit/beta-staging.test.ts index 2fb33a4..a458b3d 100644 --- a/tests/unit/beta-staging.test.ts +++ b/tests/unit/beta-staging.test.ts @@ -188,7 +188,7 @@ describe('staged beta files', () => { const { requestRestart } = await import('../../src/lib/server/provisioning/index.js'); const first = requestRestart(servers[1], { kind: 'campaign', id: older.deploymentId }); const second = requestRestart(servers[1], { kind: 'campaign', id: latest.deploymentId }); - expect(second.id).toBe(first.id); + expect(second.id).not.toBe(first.id); await activateFiles('campaign', latest.deploymentId); const { destOfKind, configFileOf } = await import('../../src/lib/server/config.js'); const relPath = kind === 'script' ? script.target_rel_path : configFileOf(servers[1]); diff --git a/tests/unit/collection-launch-performance.test.ts b/tests/unit/collection-launch-performance.test.ts index 2c883d3..9bc4b83 100644 --- a/tests/unit/collection-launch-performance.test.ts +++ b/tests/unit/collection-launch-performance.test.ts @@ -18,9 +18,7 @@ beforeEach(async () => { vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); - vi.stubEnv('KOMODO_ADDRESS', ''); - vi.stubEnv('KOMODO_API_KEY', ''); - vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Large native fleet', portMin: 28960, portMax: 29059 }).id; const inventory = { diff --git a/tests/unit/collection-match-performance.test.ts b/tests/unit/collection-match-performance.test.ts index bd60f5e..b266f92 100644 --- a/tests/unit/collection-match-performance.test.ts +++ b/tests/unit/collection-match-performance.test.ts @@ -24,9 +24,7 @@ beforeEach(async () => { vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); - vi.stubEnv('KOMODO_ADDRESS', ''); - vi.stubEnv('KOMODO_API_KEY', ''); - vi.stubEnv('KOMODO_API_SECRET', ''); + vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id; @@ -55,8 +53,8 @@ beforeEach(async () => { const instanceId = randomUUID(); const deploymentId = randomUUID(); const port = 28960 + index; - db().prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at) - VALUES(?,'','t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now); + db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at) + VALUES(?,'t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now); db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)') .run(instanceId, name, hostId, libraries[1].id, image, port, now); db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)") diff --git a/tests/unit/config-flow.test.ts b/tests/unit/config-flow.test.ts index 18479c5..ee343b9 100644 --- a/tests/unit/config-flow.test.ts +++ b/tests/unit/config-flow.test.ts @@ -10,7 +10,7 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_zombie_theater_1']); - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; }); diff --git a/tests/unit/create-server.test.ts b/tests/unit/create-server.test.ts deleted file mode 100644 index cf81f10..0000000 --- a/tests/unit/create-server.test.ts +++ /dev/null @@ -1,803 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { existsSync } from 'node:fs'; -import { readFile, rm } from 'node:fs/promises'; -import { join } from 'node:path'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { T6_ZOMBIE_MAPS } from '../../src/lib/map-aliases.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(async () => ({ ok: true })), - komodoDeploy: vi.fn(async () => ({ ok: true })), - komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), - komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), - komodoInspectHealthy: vi.fn(), - komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), - komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconGetPlayerCount: vi.fn(async () => 0), - rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })), - rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '', - status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } })) -})); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); - -let fixture: Fixture; -const serverName = 't5_zombie_cosmodrome_1'; -const input = { gameMode: 'zombie' as const, mapSlug: 'cosmodrome', plutoKey: 'test-key' }; - -beforeEach(async () => { - vi.resetModules(); - vi.clearAllMocks(); - fixture = await newFixture([]); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test'); - vi.stubEnv('KOMODO_API_SECRET', 'test'); - vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); - vi.stubEnv('RCON_PASSWORD', 'test-rcon'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult, rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rconStatusResult).mockReset().mockResolvedValue({ ok: true, - endpoint: { host: 'localhost', port: 27016 }, rawReply: '', - status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }); - vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0); -}); - -afterEach(async () => { - await fixture.cleanup(); - vi.unstubAllEnvs(); - vi.unstubAllGlobals(); -}); - -describe('queued removal cancellation', () => { - it('lets cancellation win while the worker awaits a player count, and never removes the server', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - await createServer(input); - let resolveCount!: (players: number) => void; - vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); - const queued = destroyServer(serverName); - const work = processDestructions(); - await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); - expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200); - resolveCount(0); - await work; - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Cancelled by operator' }); - expect(getDeployment(queued.deploymentId!)?.canCancelRemoval).toBe(false); - expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200); - }); - - it('does not overwrite cancellation when the in-flight player check rejects', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { cancelRemoval, getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - await createServer(input); - let rejectCount!: (error: Error) => void; - vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectCount = reject; })); - const queued = destroyServer(serverName); - const work = processDestructions(); - await vi.waitFor(() => expect(rejectCount).toBeTypeOf('function')); - expect(cancelRemoval(queued.deploymentId!)).toBe(true); - rejectCount(new Error('RCON unavailable')); - await work; - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', error: null }); - }); - - it('refuses cancellation after an atomic removal claim and for cleanup checkpoints', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { cancelRemoval, claimRemoval, canCancelRemoval, deploymentStage, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await createServer(input); - const queued = destroyServer(serverName); - expect(claimRemoval(queued.deploymentId!)).toBe(true); - expect(cancelRemoval(queued.deploymentId!)).toBe(false); - expect(claimRemoval(queued.deploymentId!)).toBe(false); - deploymentStage(queued.deploymentId!, REMOVING_SERVER); - db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(queued.deploymentId); - expect(canCancelRemoval(queued.deploymentId!)).toBe(false); - expect(cancelRemoval(queued.deploymentId!)).toBe(false); - }); -}); - -describe('destruction cleanup recovery', () => { - it('removes T6 servers while preserving closed beta progress and test evidence', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, - endpoint: { host: 'localhost', port: 28960 }, rawReply: '', - status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } }); - const name = 't6_zombie_town_1'; - expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name }); - expect(await createServer(input)).toMatchObject({ ok: true }); - const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') }); - const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'promoted')").run(script.id, version.id).lastInsertRowid; - for (const server of [name, serverName]) { - db().prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (?, ?, 0, 'ready', '2026-09-18')").run(beta, server); - db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server); - db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server); - db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server); - db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server); - } - const queued = destroyServer(name); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); - await processDestructions(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); - for (const table of ['beta_servers', 'beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) { - expect(db().prepare(`SELECT server FROM ${table} ORDER BY server`).all()).toEqual([{ server: serverName }, { server: name }]); - } - expect(db().prepare('SELECT run_id FROM beta_test_evidence ORDER BY run_id').all()).toEqual([{ run_id: serverName }, { run_id: name }]); - expect(db().prepare('SELECT stage, healthy_at, server_deleted_at FROM beta_servers WHERE server = ?').get(name)) - .toEqual({ stage: 'ready', healthy_at: '2026-09-18', server_deleted_at: expect.any(String) }); - expect(db().pragma('foreign_key_check')).toEqual([]); - }); - - it('resumes failed local cleanup after restart without RCON or another Komodo deletion', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const { POST } = await import('../../src/routes/api/deployments/[id]/resume/+server.js'); - await createServer(input); - db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'FOREIGN KEY constraint failed'); END"); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); - const { deploymentId } = destroyServer(serverName); - await processDestructions(); - expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: 'Removing server files and registry', server_exists: 1 }); - expect(getDeployment(deploymentId!)?.canResumeCleanup).toBe(true); - db().exec('DROP TRIGGER fail_cleanup'); - closeDb(); - interruptDeployments(); - vi.mocked(rconGetPlayerCount).mockClear(); - vi.mocked(komodoDestroyDeployment).mockClear(); - vi.mocked(komodoDeleteDeployment).mockClear(); - expect((await POST({ params: { id: deploymentId! } })).status).toBe(202); - expect((await POST({ params: { id: deploymentId! } })).status).toBe(409); - await processDestructions(); - expect(rconGetPlayerCount).not.toHaveBeenCalled(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); - }); - - it('reuses an interrupted cleanup when destruction is requested again', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { deploymentStage, interruptDeployments, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await createServer(input); - const first = destroyServer(serverName); - deploymentStage(first.deploymentId!, REMOVING_SERVER); - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(first.deploymentId); - interruptDeployments(); - expect(destroyServer(serverName).deploymentId).toBe(first.deploymentId); - expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER }); - }); - - it('resumes cleanup after crashing between registry deletion and the final result', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { claimRemoval, deploymentStage, interruptDeployments, getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - await createServer(input); - const { deploymentId } = destroyServer(serverName); - claimRemoval(deploymentId!); - deploymentStage(deploymentId!, REMOVING_SERVER); - db().prepare('DELETE FROM servers WHERE name = ?').run(serverName); - closeDb(); - interruptDeployments(); - expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'interrupted', server_deleted_at: expect.any(String) }); - expect(resumeCleanup(deploymentId!)).toBe(true); - await processDestructions(); - expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_deleted_at: expect.any(String), server_exists: 0 }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - }); - - it.each(['before', 'after'] as const)('protects a replacement registered %s an archived cleanup resume', async timing => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { claimRemoval, deploymentStage, interruptDeployments, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await createServer(input); - const { deploymentId } = destroyServer(serverName); - claimRemoval(deploymentId!); - deploymentStage(deploymentId!, REMOVING_SERVER); - db().prepare('DELETE FROM servers WHERE name = ?').run(serverName); - interruptDeployments(); - if (timing === 'after') expect(resumeCleanup(deploymentId!)).toBe(true); - db().prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(serverName, 'replacement'); - if (timing === 'before') expect(resumeCleanup(deploymentId!)).toBe(false); - await processDestructions(); - expect(db().prepare('SELECT komodo_name FROM servers WHERE name = ?').get(serverName)).toEqual({ komodo_name: 'replacement' }); - }); - - it('refuses cleanup before confirmed deletion or after a newer creation', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - await createServer(input); - const { deploymentId } = destroyServer(serverName); - finishDeployment(deploymentId!, { ok: false, message: 'Komodo unavailable' }); - expect(resumeCleanup(deploymentId!)).toBe(false); - deploymentStage(deploymentId!, REMOVING_SERVER); - beginDeployment(serverName, 'create'); - expect(resumeCleanup(deploymentId!)).toBe(false); - }); -}); - -describe('House Party Protocol', () => { - it('includes registered and queued servers once and reuses destruction requests', async () => { - const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); - expect(await createServer(input)).toMatchObject({ ok: true }); - expect(queueCreateServer(input)).toMatchObject({ ok: true }); - const names = housePartyTargets(); - expect(names).toEqual([serverName, 't5_zombie_cosmodrome_2']); - const first = queueHouseParty(names); - expect(first).toMatchObject({ ok: true, deployments: [{ server: serverName }, { server: names[1] }] }); - expect(queueHouseParty(names)).toEqual(first); - }); - - it('rejects a changed fleet without queuing any destruction', async () => { - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const names = housePartyTargets(); - expect(queueCreateServer(input).ok).toBe(true); - expect(queueHouseParty(names).ok).toBe(false); - expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); - }); - - it('rolls back the entire bulk request when an insert fails', async () => { - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(queueCreateServer(input).ok).toBe(true); - expect(queueCreateServer(input).ok).toBe(true); - db().exec(`CREATE TRIGGER reject_destroy BEFORE INSERT ON server_deployments - WHEN NEW.action = 'destroy' AND NEW.server = 't5_zombie_cosmodrome_2' - BEGIN SELECT RAISE(ABORT, 'test failure'); END`); - expect(() => queueHouseParty(housePartyTargets())).toThrow('test failure'); - expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); - }); - - it('waits for queued creations, then for verified emptiness, and preserves keys', async () => { - const { queueCreateServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - expect(queueCreateServer(input).ok).toBe(true); - expect(queueHouseParty(housePartyTargets()).ok).toBe(true); - await processDestructions(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - await processCreations(); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2); - await processDestructions(); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null); - await processDestructions(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(0); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never); - await processDestructions(); - expect(komodoDestroyDeployment).toHaveBeenCalledWith(serverName); - expect(getServer(serverName)).toBeUndefined(); - expect(getServerKey(serverName)).toBe('test-key'); - }); -}); - -async function promotedConfig() { - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - for (const artifact of [ - { name: 'baseline', kind: 'baseline' as const, scopeType: 'game' as const, scopeValue: 't5', - content: Buffer.from('set sv_voice "1"\nset sv_maprotation "map zombie_theater"\n') }, - { name: 'map settings', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', - content: Buffer.from('set sv_maprotation "map zombie_moon"\nset g_inactivity "0"\n') }, - { name: 'other map', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_theater', - content: Buffer.from('set sv_voice "0"\n') } - ]) { - const { script } = createScript(artifact); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') - .run(getLatestVersion(script.id)!.id, script.id); - } -} - -describe('T6 Zombies provisioning', () => { - it('keeps T6 storage and config overrides separate from T5 settings', async () => { - vi.stubEnv('T6_SERVERS_ROOT', join(fixture.root, 'custom-t6')); - vi.stubEnv('T6_CONFIG_TARGET_FILE', 'custom_zm.cfg'); - vi.stubEnv('T6_REL_DEST_SCRIPT', 'custom/scripts/zm'); - vi.stubEnv('PLUTAINER_T6_GAMEFILES_VOLUME', 'bo2-files'); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const baseline = createScript({ name: 'T6 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't6', content: Buffer.from('sv_voice 1\n') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id); - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const { destOf, configFileOf } = await import('../../src/lib/server/config.js'); - const created = await createServer({ ...input, game: 't6', mapSlug: 'town' }); - if (!created.ok) throw new Error(created.message); - const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; - expect(spec.volumes).toContain('bo2-files:/home/plutainer/gamefiles:ro'); - expect(spec.environment).toContainEqual({ variable: 'PLUTAINER_CONFIG_FILE', value: 'custom_zm.cfg' }); - expect(existsSync(join(fixture.root, 'custom-t6', created.name, 'configs/custom_zm.cfg'))).toBe(true); - expect(destOf(created.name)).toBe(`${join(fixture.root, 'custom-t6')}/${created.name}/custom/scripts/zm`); - expect(configFileOf(serverName)).toBe('dedicated_sp.cfg'); - expect(destOf(serverName)).toBe(`${fixture.t5}/${serverName}/runtime/plutonium/storage/t5/raw/scripts/sp`); - }); - - it.each(T6_ZOMBIE_MAPS)('provisions $friendly with the T6 mount, config, and ruleset', async (map) => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const name = `t6_zombie_${map.slug}_1`; - expect(await createServer({ ...input, game: 't6', mapSlug: map.slug })).toMatchObject({ ok: true, name }); - const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; - const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value])); - expect(env).toMatchObject({ PLUTAINER_GAME: 't6zm', PLUTAINER_CONFIG_FILE: 'dedicated_zm.cfg', - PLUTAINER_RCON_PASSWORD: 'test-rcon', PLUTAINER_MAP_ROTATE: 'false', - PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${name} +map_rotate` }); - expect(env.PLUTAINER_SKIP_SEED).toBeUndefined(); - expect(spec.volumes).toEqual([`${fixture.t6}/${name}:/home/plutainer/app`, 't6_game_files:/home/plutainer/gamefiles:ro']); - expect(getServer(name)).toMatchObject({ game: 't6', game_id: 't6zm', map: map.canonical, - map_slug: map.slug, config_file: 'dedicated_zm.cfg' }); - expect(await readFile(join(fixture.t6, name, 'configs/electric_cherry_map.cfg'), 'utf8')) - .toContain(`set sv_maprotation "execgts ${map.gameSettings} map ${map.canonical}"`); - expect(existsSync(join(fixture.t5, name))).toBe(false); - expect(existsSync(join(fixture.t6, name, 'configs/dedicated_zm.cfg'))).toBe(false); - }); - - it('persists T6 queued work, reuses its key, and allocates ports across both games', async () => { - const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { saveServerKey, validServerId } = await import('../../src/lib/server/fleet/server-keys.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - saveServerKey('t6_zombie_town_1', 't6-key'); - expect(validServerId('t6_zombie_town_20')).toBe(true); - expect(validServerId('t5_zombie_town_1')).toBe(false); - expect(validServerId('t6_zombie_theater_1')).toBe(false); - const t5 = queueCreateServer(input); - const t6 = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' }); - if (!t5.ok || !t6.ok) throw new Error('queue failed'); - expect(t6.port).not.toBe(t5.port); - closeDb(); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rconStatusResult).mockImplementation(async (server) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', - status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : 'zombie_cosmodrome', gametype: null, roster: [] } })); - await processCreations(); - expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); - expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); - await processCreations(); - expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'done' }); - expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); - await processCreations(); - expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'done' }); - const content = await readFile(join(fixture.t6, t6.name, 'configs/electric_cherry_map.cfg'), 'utf8'); - expect(content).toContain('execgts zm_standard_town.cfg map zm_transit'); - }); - - it('installs only matching promoted artifacts into T6 paths and removes only its directory on destruction', async () => { - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - for (const game of ['t5', 't6']) { - for (const kind of ['script', 'baseline'] as const) { - const { script, version } = createScript({ name: `${game}-${kind}`, kind, scopeType: 'game', scopeValue: game, - targetRelPath: kind === 'script' ? `${game}.gsc` : undefined, - content: Buffer.from(kind === 'script' ? 'main(){}' : 'sv_voice 1\nsv_maprotation "map wrong"\n') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); - } - } - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); - await createServer(input); - const t6 = await createServer({ ...input, game: 't6', mapSlug: 'tomb' }); - if (!t6.ok) throw new Error(t6.message); - expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t6.gsc'))).toBe(true); - expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t5.gsc'))).toBe(false); - expect(await readFile(join(fixture.t6, t6.name, 'configs/dedicated_zm.cfg'), 'utf8')) - .toContain('execgts zm_classic_tomb.cfg map zm_tomb'); - expect(await scanServerDrift(t6.name)).toEqual([]); - expect(destroyServer(t6.name).ok).toBe(true); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); - await processDestructions(); - expect(existsSync(join(fixture.t6, t6.name))).toBe(false); - expect(existsSync(join(fixture.t5, serverName))).toBe(true); - expect(getServerKey(t6.name)).toBe('test-key'); - }); - - it('validates game/map combinations at the API boundary', async () => { - const { POST } = await import('../../src/routes/api/servers/+server.js'); - for (const game of ['t7', null, 6]) { - const res = await POST({ request: new Request('http://localhost/api/servers', { - method: 'POST', body: JSON.stringify({ ...input, game }) - }) }); - expect(res.status).toBe(400); - } - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - expect(queueCreateServer({ ...input, game: 't6' })).toMatchObject({ ok: false, kind: 'unknown-map' }); - }); -}); - -describe('server creation map selection', () => { - it('returns a queued creation before provisioning and reserves its name and port', async () => { - const { POST } = await import('../../src/routes/api/servers/+server.js'); - const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - const response = await POST({ request: new Request('http://localhost/api/servers', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) - }) }); - expect(response.status).toBe(202); - const body = await response.json(); - expect(body).toMatchObject({ name: serverName, deploymentId: expect.any(String) }); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued', expected_map: 'zombie_cosmodrome' }); - expect(queueCreateServer({ ...input, expectedName: serverName })).toMatchObject({ ok: false, kind: 'no-slot' }); - const second = queueCreateServer(input); - expect(second).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); - if (second.ok) expect(second.port).not.toBe(body.port); - closeDb(); - interruptDeployments(); - expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); - await Promise.all([processCreations(), processCreations()]); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); - if (second.ok) expect(getDeployment(second.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - }); - - it('completes creation only when Komodo, RCON, and the selected map all pass', async () => { - const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const created = await createServer(input); - if (!created.ok) throw new Error('fixture creation failed'); - const id = created.deploymentId; - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); - await processCreations(); - expect(rconStatusResult).not.toHaveBeenCalled(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' }); - await processCreations(); - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON or fresh telemetry health: timeout' }); - for (const map of [null, 'zombie_moon']) { - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); - if (created.port === null) throw new Error('A provisioned legacy server must have a port.'); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } }); - await processCreations(); - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') }); - } - interruptDeployments(); - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); - await processCreations(); - expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, map zombie_cosmodrome, and Plutonium r5354 (production r5354)' }); - }); - - it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); - const { GET } = await import('../../src/routes/api/deployments/[id]/+server.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - await createServer(input); - const response = await DELETE({ params: { name: serverName } }); - expect(response.status).toBe(202); - const body = await response.json(); - expect(body).toMatchObject({ ok: true, deploymentId: expect.any(String) }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - const detail = await (await GET({ params: { id: body.deploymentId } })).json(); - expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' }); - expect(detail.events.map((event: { message: string }) => event.message)).toEqual(expect.arrayContaining([ - 'Server destruction requested', 'Waiting to verify the server is empty' - ])); - expect(detail.events.every((event: { id: number }, index: number) => index === 0 || detail.events[index - 1].id < event.id)).toBe(true); - const missing = await DELETE({ params: { name: 'missing-server' } }); - expect(missing.status).toBe(404); - }); - - it('queues destruction, waits for live zero players, and preserves an ordered timeline', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - await createServer(input); - const queued = destroyServer(serverName); - expect(queued).toMatchObject({ ok: true, deploymentId: expect.any(String) }); - expect(destroyServer(serverName).deploymentId).toBe(queued.deploymentId); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3).mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('RCON unavailable')); - for (let i = 0; i < 3; i++) { - await processDestructions(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); - } - closeDb(); - interruptDeployments(); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); - await Promise.all([processDestructions(), processDestructions()]); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - const detail = getDeployment(queued.deploymentId!); - expect(detail?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); - const messages = (detail?.events as { message: string }[]).map(e => e.message); - expect(messages).toEqual(expect.arrayContaining(['Server destruction requested', 'Waiting for server to empty: 3 player(s) online', 'Server is empty (0 players verified)', 'Stopping and removing container', 'Completed'])); - expect(messages.indexOf('Server is empty (0 players verified)')).toBeLessThan(messages.indexOf('Stopping and removing container')); - }); - - it('preserves files and registry when Komodo rejects destruction and exposes failure details', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - await createServer(input); - vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'denied' }); - const queued = destroyServer(serverName); - await processDestructions(); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - expect(existsSync(join(fixture.t5, serverName))).toBe(true); - expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', server_exists: 1, error: 'DestroyDeployment failed: denied' }); - }); - - it('records in-flight creation and retains both operations after destruction', async () => { - const { createServer, destroyServer, processDestructions, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDeploy).mockImplementationOnce(async () => { - expect(listDeployments()).toContainEqual(expect.objectContaining({ server: serverName, action: 'create', state: 'running', stage: 'Starting container' })); - return { ok: true }; - }); - expect((await createServer(input)).ok).toBe(true); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); - await processCreations(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); - vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { - expect(listDeployments()).toContainEqual(expect.objectContaining({ action: 'destroy', state: 'running' })); - return { ok: true }; - }); - expect((await destroyServer(serverName)).ok).toBe(true); - await processDestructions(); - expect(listDeployments()).toEqual(expect.arrayContaining([ - expect.objectContaining({ server: serverName, action: 'create', state: 'done', server_exists: 0 }), - expect.objectContaining({ server: serverName, action: 'destroy', state: 'done', server_exists: 0 }) - ])); - }); - - it('records failed and unexpectedly interrupted creation attempts', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { listDeployments, beginDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); - expect((await createServer(input)).ok).toBe(false); - expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' })); - vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost')); - await expect(createServer(input)).rejects.toThrow('connection lost'); - expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'awaiting-health', - stage: 'Verifying existing server after interrupted startup', finished_at: null })); - const id = beginDeployment('interrupted-server', 'destroy'); - interruptDeployments(); - expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) })); - }); - - it('installs only promoted scope-matching scripts and plugins before first boot', async () => { - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { destOfKind } = await import('../../src/lib/server/config.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); - const artifacts = [ - { kind: 'script' as const, target: 'zm_spawn_fix.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: true, expected: true }, - { kind: 'plugin' as const, target: 'plugin.dll', scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', promoted: true, expected: true }, - { kind: 'script' as const, target: 'other.gsc', scopeType: 'map' as const, scopeValue: 'zombie_theater', promoted: true, expected: false }, - { kind: 'script' as const, target: 'draft.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: false, expected: false } - ].map((a) => { - const { script, version } = createScript({ name: a.target, kind: a.kind, targetRelPath: a.target, - scopeType: a.scopeType, scopeValue: a.scopeValue, content: Buffer.from(a.target) }); - if (a.promoted) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); - return { ...a, version }; - }); - vi.mocked(komodoDeploy).mockImplementationOnce(async () => { - for (const a of artifacts) { - const path = join(destOfKind(serverName, a.kind), a.target); - expect(existsSync(path)).toBe(a.expected); - if (a.expected) { - expect(await readFile(path)).toEqual(a.version.content); - expect(getServerScript(serverName, a.target)).toMatchObject({ script_version_id: a.version.id, source: 'create' }); - } - } - return { ok: true }; - }); - expect(await createServer(input)).toMatchObject({ ok: true }); - expect(await scanServerDrift(serverName)).toEqual([]); - }); - - it('keeps Plutainer defaults without a baseline and overrides the map before rotation', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const result = await createServer(input); - expect(result).toMatchObject({ ok: true, name: serverName }); - expect(existsSync(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'))).toBe(false); - const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; - const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value])); - expect(env.PLUTAINER_MAP_ROTATE).toBe('false'); - expect(env.PLUTAINER_EXTRA_ARGS).toBe(`+exec electric_cherry_map.cfg +set tws_server_id ${serverName} +map_rotate`); - expect(await readFile(join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'), 'utf8')) - .toContain('set sv_maprotation "map zombie_cosmodrome"'); - expect(env.PLUTAINER_CONFIG_FILE).toBe('dedicated_sp.cfg'); - expect(env.PLUTAINER_SKIP_SEED).toBeUndefined(); - expect(komodoDeploy).toHaveBeenCalledWith(serverName); - expect(getServer(serverName)?.map).toBe('zombie_cosmodrome'); - const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); - expect(await scanServerDrift(serverName)).toEqual([]); - const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'); - await rm(mapPath); - const repair = await runDriftDeployment({ servers: [serverName] }); - expect(repair.totalRestored).toBe(1); - expect(existsSync(mapPath)).toBe(false); - const { activateFiles } = await import('../helpers/staged-files.js'); - await activateFiles('campaign', repair.deploymentId!); - expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"'); - expect(await scanServerDrift(serverName)).toEqual([]); - }); - - it('writes promoted settings and selected map before first boot and tracks the file', async () => { - await promotedConfig(); - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); - vi.mocked(komodoDeploy).mockImplementationOnce(async () => { - const config = await readFile(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'), 'utf8'); - expect(config).toContain('set sv_maprotation "map zombie_cosmodrome"'); - expect(config).toContain('set sv_voice "1"'); - expect(config).toContain('set g_inactivity "0"'); - expect(config).not.toContain('map zombie_moon'); - expect(getServerScript(serverName, 'dedicated_sp.cfg')?.source).toBe('create'); - return { ok: true }; - }); - expect(await createServer(input)).toMatchObject({ ok: true }); - expect(await scanServerDrift(serverName)).toEqual([]); - }); - - it('removes the generated config and inventory if deployment fails', async () => { - await promotedConfig(); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { script, version } = createScript({ name: 'spawn fix', kind: 'script', scopeType: 'game', scopeValue: 't5', - targetRelPath: 'zm_spawn_fix.gsc', content: Buffer.from('main(){}') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); - expect(await createServer(input)).toMatchObject({ ok: false, kind: 'komodo' }); - expect(getServer(serverName)).toBeUndefined(); - expect(getServerScript(serverName, 'dedicated_sp.cfg')).toBeUndefined(); - expect(getServerScript(serverName, 'zm_spawn_fix.gsc')).toBeUndefined(); - expect(existsSync(join(fixture.t5, serverName))).toBe(false); - }); - - it('rejects unknown maps before creating a deployment', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - expect(await createServer({ ...input, mapSlug: 'unknown' })).toMatchObject({ ok: false, kind: 'unknown-map' }); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - }); -}); - - -describe('persistent server keys', () => { - it('saves a key, retains it through destruction, and reuses it for the exact lobby', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const name = 't5_zombie_cosmodrome_2'; - expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); - expect(await createServer(input)).toMatchObject({ ok: true, name }); - expect(getServerKey(name)).toBe('test-key'); - expect(await destroyServer(name)).toMatchObject({ ok: true }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); - await processDestructions(); - expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 })); - expect(JSON.stringify(listServerKeys())).not.toContain('test-key'); - expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: name })).toMatchObject({ ok: true, name }); - const spec = vi.mocked(komodoCreateDeployment).mock.calls.at(-1)![0]; - expect(spec.environment).toContainEqual({ variable: 'PLUTO_SERVER_KEY', value: 'test-key' }); - expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome' })).toMatchObject({ ok: false, kind: 'bad-request' }); - }); - - it('updates and removes keys independently of deployments and survives reopening the database', async () => { - const { saveServerKey, getServerKey, removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - saveServerKey(serverName, ' first-key '); - closeDb(); - expect(getServerKey(serverName)).toBe('first-key'); - saveServerKey(serverName, 'replacement'); - expect(getServerKey(serverName)).toBe('replacement'); - expect(removeServerKey(serverName)).toBe(true); - expect(getServerKey(serverName)).toBeUndefined(); - expect(() => saveServerKey('t5_zombie_unknown_1', 'key')).toThrow(); - expect(() => saveServerKey(serverName, ' ')).toThrow(); - expect(() => saveServerKey(serverName, 'two words')).toThrow(); - }); - - it('fills the lowest gap and rejects stale previews before saving a key or deploying', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); - expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); - await destroyServer(serverName); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); - await processDestructions(); - expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' })) - .toMatchObject({ ok: false, kind: 'no-slot' }); - expect(getServerKey(serverName)).toBe('test-key'); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: serverName })) - .toMatchObject({ ok: true, name: serverName }); - }); -}); - -describe('cancelling submitted creation and destruction', () => { - it('retains a remotely created identity after cancellation without starting its container', async () => { - const { queueCreateServer, processCreations, nextAvailablePort, nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const queued = queueCreateServer(input); if (!queued.ok) throw new Error(queued.message); - let release!: (value: {ok:true}) => void; - vi.mocked(komodoCreateDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;})); - const work = processCreations(); - await vi.waitFor(() => expect(release).toBeTypeOf('function')); - expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true); - expect(nextAvailablePort()).not.toBe(queued.port); - expect(nextLobbyNumber('t5_zombie_cosmodrome')).toBe(2); - release({ok:true}); await work; - expect(getServer(queued.name)).toBeTruthy(); - expect(komodoDeploy).not.toHaveBeenCalled(); - expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled'); - }); - - it('does not delete deployment metadata or files after cancelling container teardown', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js'); - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - await createServer(input); - let release!: (value: {ok:true}) => void; - vi.mocked(komodoDestroyDeployment).mockImplementationOnce(() => new Promise(resolve => {release=resolve;})); - const queued=destroyServer(serverName); const work=processDestructions(); - await vi.waitFor(() => expect(release).toBeTypeOf('function')); - expect(cancelProvisioningJob(queued.deploymentId!)).toBe(true); - release({ok:true}); await work; - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - expect(getServer(serverName)).toBeTruthy(); - expect(existsSync(join(fixture.t5,serverName))).toBe(true); - expect(getDeployment(queued.deploymentId!)?.deployment.state).toBe('cancelled'); - }); -}); diff --git a/tests/unit/creation-queue.test.ts b/tests/unit/creation-queue.test.ts deleted file mode 100644 index 95457c3..0000000 --- a/tests/unit/creation-queue.test.ts +++ /dev/null @@ -1,403 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(), - komodoDeploy: vi.fn(), - komodoDestroyDeployment: vi.fn(), - komodoDeleteDeployment: vi.fn(), - komodoInspectHealthy: vi.fn(), - komodoEnsureTag: vi.fn(), - komodoUpdateTagsOnResource: vi.fn() -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconGetPlayerCount: vi.fn(), - rconPlutoniumVersion: vi.fn(), - rconStatusResult: vi.fn() -})); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); - -let fixture: Fixture; -const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'test-key' }; -function status(server: string, map = server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater') { - return { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', - status: { players: 0, map, gametype: null, roster: [] } }; -} - -beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); - fixture = await newFixture([]); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test-key'); - vi.stubEnv('KOMODO_API_SECRET', 'test-secret'); - vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); - vi.stubEnv('RCON_PASSWORD', 'test-password'); - vi.stubEnv('SERVER_PORT_MIN', '28960'); - vi.stubEnv('SERVER_PORT_MAX', '28980'); - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodo.komodoCreateDeployment).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoDeploy).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoDeleteDeployment).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name })); - vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true }); - const rcon = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0); - vi.mocked(rcon.rconPlutoniumVersion).mockImplementation(async () => { - // Version evidence must be observed strictly after the launch timestamp. - await new Promise(resolve => setTimeout(resolve, 2)); - return { revision: 5354, error: null }; - }); - vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server)); - const { kick } = await import('../../src/lib/server/pipeline/worker.js'); - vi.mocked(kick).mockResolvedValue(undefined); -}); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.restoreAllMocks(); }); - -async function queueServers(count: number) { - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - return Array.from({ length: count }, () => { - const result = queueCreateServer(input); - if (!result.ok) throw new Error(result.message); - return result; - }); -} -async function states() { - const { db } = await import('../../src/lib/server/db/index.js'); - return (db().prepare("SELECT state FROM server_deployments WHERE action = 'create' ORDER BY rowid").all() as { state: string }[]).map(row => row.state); -} - -describe('shared creation queue', () => { - it('rechecks a lowered limit after awaiting a startup in the same pass', async () => { - await queueServers(3); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { - saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); - return { ok: true }; - }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']); - }); - - it('applies live increases and decreases while preserving active startups', async () => { - const jobs = await queueServers(4); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoInspectHealthy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'still starting' }); - await processCreations(); - saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); - await processCreations(); - expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']); - saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(3); - expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'awaiting-health', 'queued']); - vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[2].name - ? { ok: false, reason: 'still starting' } : { ok: true, state: 'running' }); - await processCreations(); - expect(await states()).toEqual(['done', 'done', 'awaiting-health', 'queued']); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - await processCreations(); - expect(await states()).toEqual(['done', 'done', 'done', 'awaiting-health']); - }); - - it('starts only one server per pass and holds its slot until a later health check', async () => { - const jobs = await queueServers(3); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoInspectHealthy).not.toHaveBeenCalled(); - expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']); - await processCreations(); - expect(komodoInspectHealthy).toHaveBeenCalledWith(jobs[0].name); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(await states()).toEqual(['done', 'awaiting-health', 'queued']); - expect(new Set(jobs.map(job => job.port)).size).toBe(3); - }); - - it('keeps later boots queued through container, RCON, and expected-map failures', async () => { - const jobs = await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - await processCreations(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); - await processCreations(); - expect(rconStatusResult).not.toHaveBeenCalled(); - expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' }); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' }); - await processCreations(); - expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON or fresh telemetry health: timeout' }); - vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon')); - await processCreations(); - expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') }); - vi.mocked(komodoInspectHealthy).mockRejectedValueOnce(new Error('Temporary connection error')); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(await states()).toEqual(['awaiting-health', 'queued']); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(await states()).toEqual(['done', 'awaiting-health']); - }); - - it('shares FIFO capacity across collections and direct callers, including equal timestamps', async () => { - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const collections = ['theater', 'moon'].map(mapSlug => { - const blueprint = saveBlueprint({ name: mapSlug, entries: [{ game: 't5', mapSlug, count: 2 }] }); - const preview = previewCollection(blueprint.id); - return queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name, 'test-key'])) }); - }); - db().prepare("UPDATE server_deployments SET started_at = '2026-01-01T00:00:00.000Z'").run(); - const direct = await createServer(input); - if (!direct.ok) throw new Error(direct.message); - expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued', 'queued']); - const second = getCollection(collections[1].collectionId!)!; - expect(second.collection).toMatchObject({ queued: 2, provisioning: 0, verifying: 0 }); - expect(second.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: 't5_zombie_theater_1' }] }); - expect(second.servers.map(server => server.queuePosition)).toEqual([2, 3]); - for (let pass = 0; pass < 5; pass++) await processCreations(); - expect(vi.mocked(komodoCreateDeployment).mock.calls.map(([spec]) => spec.name)).toEqual([ - 't5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't5_zombie_moon_2', direct.name - ]); - expect(await states()).toEqual(['done', 'done', 'done', 'done', 'done']); - }); - - it('coalesces overlapping worker passes and direct calls while provisioning is pending', async () => { - await queueServers(3); - const { processCreations, createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - let release!: () => void; - let entered!: () => void; - const blocked = new Promise(resolve => { release = resolve; }); - const started = new Promise(resolve => { entered = resolve; }); - vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); - const first = processCreations(); - await started; - const second = processCreations(); - const direct = createServer(input); - expect(await states()).toEqual(['running', 'queued', 'queued', 'queued']); - release(); - await Promise.all([first, second, direct]); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued']); - }); - - it('honors a higher limit without refilling its pass budget after immediate failures', async () => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); - await queueServers(5); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: false, kind: 'http-error', message: 'Host unavailable' }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(await states()).toEqual(['failed', 'failed', 'queued', 'queued', 'queued']); - vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: true }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(4); - expect(await states()).toEqual(['failed', 'failed', 'awaiting-health', 'awaiting-health', 'queued']); - }); - - it('does not provision a selected job that was cancelled during an earlier startup', async () => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); - const jobs = await queueServers(3); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - let release!: () => void; - let entered!: () => void; - const blocked = new Promise(resolve => { release = resolve; }); - const started = new Promise(resolve => { entered = resolve; }); - vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); - const pass = processCreations(); - await started; - finishDeployment(jobs[1].deploymentId, { ok: false, message: 'Cancelled before startup' }); - release(); - await pass; - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(await states()).toEqual(['awaiting-health', 'failed', 'queued']); - }); - - it('rechecks capacity when an existing server resumes verification during provisioning', async () => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); - const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { finishDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const old = await createServer(input); - if (!old.ok) throw new Error(old.message); - finishDeployment(old.deploymentId, { ok: false, message: 'Needs verification' }); - await queueServers(2); - let release!: () => void; - let entered!: () => void; - const blocked = new Promise(resolve => { release = resolve; }); - const started = new Promise(resolve => { entered = resolve; }); - vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); - const pass = processCreations(); - await started; - expect(resumeVerification(old.deploymentId)).toBe(true); - release(); - await pass; - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'queued']); - }); - - it('counts already verifying servers against a higher concurrency limit', async () => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); - const jobs = await queueServers(4); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - await processCreations(); - vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[0].name - ? { ok: false, reason: 'still booting' } : { ok: true, state: 'running' }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(3); - expect(await states()).toEqual(['awaiting-health', 'done', 'awaiting-health', 'queued']); - }); - - it('retains durable reservations and the startup gate across an application restart', async () => { - const jobs = await queueServers(3); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - await processCreations(); - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId); - closeDb(); - vi.resetModules(); - const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { processCreations: resumed, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js'); - interruptDeployments(); - expect(await states()).toEqual(['interrupted', 'queued', 'queued']); - expect(nextAvailablePort()).toBe(28963); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); - await resumed(); - expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - await resumed(); - expect(await states()).toEqual(['done', 'awaiting-health', 'queued']); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - }); - - it('keeps an ambiguous startup response in verification without replaying its deployment', async () => { - await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDeploy, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('Response lost after starting')); - const log = vi.spyOn(console, 'error').mockImplementation(() => {}); - await processCreations(); - expect(await states()).toEqual(['awaiting-health', 'queued']); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - expect(log).toHaveBeenCalled(); - await processCreations(); - expect(await states()).toEqual(['done', 'awaiting-health']); - }); - - it.each([ - { ok: false as const, kind: 'unreachable' as const, message: 'Request timed out after Komodo accepted it' }, - { ok: false as const, kind: 'http-error' as const, httpStatus: 502, message: 'Gateway lost the startup response' } - ])('holds the slot and files for an uncertain structured response: $kind', async failure => { - const jobs = await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoDeploy, komodoDeleteDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDeploy).mockResolvedValueOnce(failure); - await processCreations(); - expect(await states()).toEqual(['awaiting-health', 'queued']); - expect(getServer(jobs[0].name)?.game_port).toBe(jobs[0].port); - expect(existsSync(join(fixture.t5, jobs[0].name, 'configs/electric_cherry_map.cfg'))).toBe(true); - expect(komodoDeleteDeployment).not.toHaveBeenCalled(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - await processCreations(); - expect(await states()).toEqual(['done', 'awaiting-health']); - }); - - it('releases capacity on a confirmed deployment rejection', async () => { - const jobs = await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoDeploy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', httpStatus: 403, message: 'Access denied' }); - await processCreations(); - expect(await states()).toEqual(['failed', 'queued']); - expect(getServer(jobs[0].name)).toBeUndefined(); - expect(existsSync(join(fixture.t5, jobs[0].name))).toBe(false); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - await processCreations(); - expect(await states()).toEqual(['failed', 'awaiting-health']); - }); - - it('retains an interrupted startup slot while a newer destruction is still waiting', async () => { - const jobs = await queueServers(2); - const { processCreations, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - await processCreations(); - expect(destroyServer(jobs[0].name)).toMatchObject({ ok: true }); - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId); - interruptDeployments(); - expect(await states()).toEqual(['interrupted', 'queued']); - await processCreations(); - expect(await states()).toEqual(['awaiting-health', 'queued']); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoInspectHealthy).not.toHaveBeenCalled(); - }); - - it('releases a startup slot after its server is removed', async () => { - const jobs = await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await processCreations(); - db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name); - await processCreations(); - expect(await states()).toEqual(['cancelled', 'awaiting-health']); - }); - - it('never verifies a replacement registered under the same server name', async () => { - const jobs = await queueServers(2); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - await processCreations(); - db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(28980, jobs[0].name); - await processCreations(); - expect(komodoInspectHealthy).not.toHaveBeenCalledWith(jobs[0].name); - expect(await states()).toEqual(['failed', 'awaiting-health']); - }); -}); - -describe('creation concurrency configuration', () => { - it('defaults to one startup', async () => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined); - const { fleetSettings } = await import('../../src/lib/server/settings.js'); - expect(fleetSettings().createConcurrency).toBe(1); - }); - - it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => { - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value); - const { fleetSettings } = await import('../../src/lib/server/settings.js'); - expect(() => fleetSettings()).toThrow(/Concurrent server startups/); - vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); - }); -}); diff --git a/tests/unit/deleted-server-guards.test.ts b/tests/unit/deleted-server-guards.test.ts index e59768e..7877749 100644 --- a/tests/unit/deleted-server-guards.test.ts +++ b/tests/unit/deleted-server-guards.test.ts @@ -14,7 +14,7 @@ const path = (server: string) => join(fixture.t5, server, 'runtime/plutonium/sto beforeEach(async () => { vi.resetModules(); fixture = await newFixture(names); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', ''); + vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); diff --git a/tests/unit/deployment-collections.test.ts b/tests/unit/deployment-collections.test.ts index dad8cba..85a7fd3 100644 --- a/tests/unit/deployment-collections.test.ts +++ b/tests/unit/deployment-collections.test.ts @@ -1,18 +1,10 @@ +import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import type { CollectionPreview } from '../../src/lib/deployment-plan.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(async () => ({ ok: true })), - komodoDeploy: vi.fn(async () => ({ ok: true })), - komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), - komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), - komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })), - komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), - komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => { @@ -34,14 +26,10 @@ const entries = [ beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); + await prepareNativeHost(); vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'private-api-key'); - vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret'); - vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); + vi.stubEnv('RCON_PASSWORD', 'private-rcon-password'); - vi.stubEnv('SERVER_PORT_MIN', '28960'); - vi.stubEnv('SERVER_PORT_MAX', '28980'); vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); @@ -67,26 +55,6 @@ describe('multi-server blueprints', () => { expect(getCollection(result.collectionId!)?.queue.concurrency).toBe(3); }); - it('saves a mixed-game lineup and previews unique slots and global ports without reserving them', async () => { - const { blueprint, preview } = await previewLineup(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); - expect(blueprint.totalServers).toBe(5); - expect(blueprint.entries.map(entry => entry.count)).toEqual([2, 1, 2]); - expect(blueprint).not.toHaveProperty('recipe'); - const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string }; - expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries }); - expect(preview.ready).toBe(true); - expect(preview.launchConcurrency).toBe(1); - expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']); - expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]); - expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); - expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]); - expect(JSON.stringify(preview)).not.toContain('private-'); - expect(preview.entries[0].recipe.files[0]).not.toHaveProperty('content'); - expect(() => previewCreation({ blueprintId: blueprint.id })).toThrow(/multiple servers/); - }); - it('reads a legacy one-server blueprint as a map selection and saves edits without snapshots', async () => { const { captureRecipe, listBlueprints, updateBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); @@ -112,149 +80,15 @@ describe('multi-server blueprints', () => { expect(listBlueprints()).toHaveLength(1); }); - it('uses current promotions for legacy collection blueprints while retaining counts and exact T6 variants', async () => { - const { captureRecipe, listBlueprints } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const artifact = createScript({ name: 'Legacy telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id); - const legacyEntries = entries.map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipe(game as 't5' | 't6', mapSlug) })); - db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy-lineup', 'Legacy lineup', new Date().toISOString(), JSON.stringify({ format: 2, entries: legacyEntries })); - const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id); - closeDb(); - expect(listBlueprints()[0].entries.map(({ mapName: _mapName, ...entry }) => entry)).toEqual(entries); - const preview = previewCollection('legacy-lineup'); - expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id })); - expect(preview.entries[2].recipe.mapSlug).toBe('town'); - const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 }); - for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { - expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry'); - } - expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); - }); - it.each([[], [{ game: 't5', mapSlug: 'theater', count: 0 }], [{ game: 't5', mapSlug: 'theater', count: 21 }], [{ game: 't5', mapSlug: 'theater', count: 1.5 }], [{ game: 't5', mapSlug: 'theater', count: '2' }], [{ game: 't5', mapSlug: 'town', count: 1 }], [entries[0], entries[0]]].map(input => ({ input })))('rejects an invalid lineup: $input', async ({ input }) => { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); expect(() => saveBlueprint({ name: 'Invalid', entries: input })).toThrow(); }); - - it('reports insufficient capacity without preventing blueprint creation', async () => { - vi.stubEnv('SERVER_PORT_MAX', '28961'); - const { blueprint, preview } = await previewLineup(); - const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(blueprint.totalServers).toBe(5); - expect(preview.ready).toBe(false); - expect(preview.servers.filter(server => server.port === null)).toHaveLength(3); - expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/); - expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); - }); }); describe('atomic collection launches', () => { - it('fills the target shortfall and counts registered/in-flight identities only once', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - for (let i = 0; i < 2; i++) expect((await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' })).ok).toBe(true); - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const blueprint = saveBlueprint({ name: 'Friday targets', entries: [ - { game: 't5', mapSlug: 'theater', count: 4 }, { game: 't5', mapSlug: 'factory', count: 2 } - ] }); - const preview = previewCollection(blueprint.id); - expect(preview.entries.map(({ count, existing, missing }) => ({ count, existing, missing }))).toEqual([ - { count: 4, existing: 2, missing: 2 }, { count: 2, existing: 0, missing: 2 } - ]); - expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_3', 't5_zombie_theater_4', 't5_zombie_factory_1', 't5_zombie_factory_2']); - const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - expect(getCollection(queued.collectionId!)?.collection.total).toBe(4); - expect(previewCollection(blueprint.id).servers).toEqual([]); - }); - - it('keeps excess and unrelated servers, including stopped servers, without requiring launch settings for a no-op', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' }); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE servers SET container_state = 'stopped'").run(); - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { config } = await import('../../src/lib/server/config.js'); - // Fixture-only configuration changes simulate missing integrations after creation. - Object.assign(config.rcon, { password: '' }); - Object.assign(config.komodo, { targetServer: '' }); - const blueprint = saveBlueprint({ name: 'One Kino', entries: [{ game: 't5', mapSlug: 'theater', count: 1 }] }); - const preview = previewCollection(blueprint.id); - expect(preview.entries[0]).toMatchObject({ existing: 2, missing: 0 }); - expect(preview).toMatchObject({ ready: true, blockers: [], servers: [] }); - expect(queueCollection({ previewId: preview.id })).toEqual({ ok: true, collectionId: null }); - expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); - expect(db().prepare('SELECT * FROM servers').all()).toHaveLength(2); - }); - - it('rechecks shrinking deficits at submission and rejects an older additive preview', async () => { - const { preview, blueprint } = await previewLineup(); - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { queueCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' }); - expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/changed/); - expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1); - const fresh = previewCollection(blueprint.id); - expect(fresh.servers).toHaveLength(4); - const extra = [...fresh.servers, { ...fresh.servers[0], name: 't5_zombie_theater_20', port: 28980 }]; - db().prepare('UPDATE server_collection_previews SET servers = ? WHERE id = ?').run(JSON.stringify(extra), fresh.id); - expect(() => queueCollection({ previewId: fresh.id, keys: keysFor(fresh) })).toThrow(/changed/); - }); - - it('matches exact T6 variants and excludes failed creations and pending destructions', async () => { - const { createServer, queueCreateServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { previewCollection } = await import('../../src/lib/server/fleet/collections.js'); - const farm = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm-key' }); - const town = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town-key' }); - if (!farm.ok || !town.ok) throw new Error('Fixture creation failed'); - destroyServer(town.name); - const failed = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'failed-key' }); - if (!failed.ok) throw new Error(failed.message); - finishDeployment(failed.deploymentId, { ok: false, message: 'Fixture failure' }); - const blueprint = saveBlueprint({ name: 'Town target', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] }); - const preview = previewCollection(blueprint.id); - expect(preview.entries[0]).toMatchObject({ existing: 0, missing: 1 }); - expect(preview.servers[0].name).toBe('t6_zombie_town_2'); - }); - - it('queues all five servers, persists one group, and completes each independently', async () => { - const { preview } = await previewLineup(); - const { queueCollection, getCollection, listCollections } = await import('../../src/lib/server/fleet/collections.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 }); - expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]); - closeDb(); - await processCreations(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 }); - expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] }); - expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]); - for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); - const detail = getCollection(queued.collectionId!)!; - expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 }); - expect(listCollections()).toHaveLength(1); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(5); - expect(getDeployment(detail.servers[0].id)?.collection).toEqual({ id: queued.collectionId, name: 'Friday Zombies' }); - expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); - expect(JSON.stringify(detail)).not.toContain('key-t5'); - }); it('rolls back all earlier jobs and key changes if the final server has no valid key', async () => { const { preview } = await previewLineup(); @@ -264,7 +98,7 @@ describe('atomic collection launches', () => { saveServerKey(preview.servers[0].name!, 'original-key'); const keys = keysFor(preview); delete keys[preview.servers[4].name!]; - expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/No saved key/); + expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/Save a Plutonium server key/); for (const table of ['server_launch_collections', 'server_collection_members', 'server_creation_jobs', 'server_deployments', 'server_launch_previews', 'server_launch_recipes']) { expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); } @@ -276,18 +110,6 @@ describe('atomic collection launches', () => { expect(queueCollection({ previewId: preview.id, keys: keysFor(preview) }).ok).toBe(true); }); - it('rejects changed capacity before saving keys or partially queueing the lineup', async () => { - const { preview } = await previewLineup(); - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - queueCreateServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', plutoKey: 'other-server-key' }); - expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/availability changed/); - expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1); - expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1); - expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); - }); - it('reuses saved keys and returns the same collection on resubmission after restart and expiry', async () => { const { preview, blueprint } = await previewLineup(); const { queueCollection, previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); @@ -309,52 +131,6 @@ describe('atomic collection launches', () => { expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1); }); - it('keeps the reviewed lineup after blueprint edits and deletion while using current promotions', async () => { - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { script, version } = createScript({ name: 'Classic perks', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'perks.gsc', content: Buffer.from('original') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); - const { preview, blueprint } = await previewLineup(); - const { updateBlueprint, deleteBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const newer = uploadNewVersion(script.id, Buffer.from('new version')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newer.id, script.id); - updateBlueprint(blueprint.id, { name: 'Changed', entries: [{ ...entries[0], count: 1 }] }); - deleteBlueprint(blueprint.id); - const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - const queuedDetail = getCollection(queued.collectionId!)!; - const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); - for (const server of queuedDetail.servers.filter(server => server.server.startsWith('t5_'))) { - expect(recipeForDeployment(server.id)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newer.id })); - } - const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id); - uploadNewVersion(script.id, Buffer.from('unpromoted draft')); - for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 }); - for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { - expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued'); - } - }); - - it('keeps a provisioning failure separate while the other servers complete', async () => { - const { preview } = await previewLineup(); - const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' }); - const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - await processCreations(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 }); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); - const detail = getCollection(queued.collectionId!)!; - expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 }); - expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') }); - expect(detail.servers.slice(1).every(server => server.state === 'done')).toBe(true); - }); - it('validates collection API payloads and rejects an unused expired preview', async () => { const { preview, blueprint } = await previewLineup(); const { POST: singlePreview } = await import('../../src/routes/api/deployments/preview/+server.js'); @@ -382,52 +158,6 @@ describe('matching a blueprint exactly', () => { return saveBlueprint({ name: 'Exact lineup', entries }); } - it('keeps the lowest deployed lobbies, removes excess and unrelated maps, and adds only the shortfall', async () => { - const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - for (const mapSlug of ['theater', 'theater', 'moon']) await createServer({ gameMode: 'zombie', mapSlug, plutoKey: 'existing' }); - const { db } = await import('../../src/lib/server/db/index.js'); - const before = db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get(); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }, { game: 't6', mapSlug: 'town', count: 1 }]); - const preview = previewCollection(plan.id, 'exact'); - expect(preview.retainedServers).toEqual(['t5_zombie_theater_1']); - expect(preview.removals.map(server => server.name)).toEqual(['t5_zombie_moon_1', 't5_zombie_theater_2']); - expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_1']); - const { komodoDestroyDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockClear(); - const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getCollection(result.collectionId!)?.collection).toMatchObject({ mode: 'exact', total: 3, active: 3, queued: 3 }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }).mockResolvedValueOnce({ ok: false, reason: 'gone' }); - await processDestructions(); - await processCreations(); - await processCreations(); - expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 3, done: 3, active: 0 }); - expect(getCollection(result.collectionId!)?.servers.filter(server => server.action === 'destroy')).toHaveLength(2); - expect(db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get()).toEqual(before); - expect(db().prepare('SELECT name FROM servers ORDER BY name').all()).toEqual([{ name: 't5_zombie_theater_1' }, { name: 't6_zombie_town_1' }]); - expect(komodoDestroyDeployment).not.toHaveBeenCalledWith('t5_zombie_theater_1'); - }); - - it('supports removal-only plans, shares existing removals, and records retained servers', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const first = previewCollection(plan.id, 'exact'); - expect(first.servers).toEqual([]); - const a = queueCollection({ previewId: first.id }); - const b = queueCollection({ previewId: previewCollection(plan.id, 'exact').id }); - const firstDetail = getCollection(a.collectionId!)!; - expect(firstDetail.retainedServers).toEqual(['t5_zombie_theater_1']); - expect(firstDetail.servers).toHaveLength(1); - expect(getCollection(b.collectionId!)?.servers[0].id).toBe(firstDetail.servers[0].id); - expect(firstDetail.servers[0]).toMatchObject({ action: 'destroy', state: 'waiting', queuePosition: null }); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - expect(getDeployment(firstDetail.servers[0].id)?.collection).toBeTruthy(); - }); - it('cancels unstarted excess creations without booting them and retains their saved keys', async () => { const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const first = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'first' }); @@ -442,21 +172,6 @@ describe('matching a blueprint exactly', () => { expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 }); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(second.name)).toBe('second'); - const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - - it('rejects fleet changes even when the create shortfall stays the same', async () => { - const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'new' }); - expect(() => queueCollection({ previewId: preview.id })).toThrow(/fleet changed/); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); it('rolls back removals and creation cancellations when a required key is missing', async () => { @@ -466,128 +181,13 @@ describe('matching a blueprint exactly', () => { const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); - expect(() => queueCollection({ previewId: preview.id })).toThrow(/No saved key/); + expect(() => queueCollection({ previewId: preview.id })).toThrow(/Save a Plutonium server key/); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' }); for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); - it('reuses a full fleet port only after empty-server removal, across restart and repeated submission', async () => { - vi.stubEnv('SERVER_PORT_MAX', '28960'); - const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); - await processCreations(); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - expect(preview).toMatchObject({ ready: true, servers: [{ port: 28960 }] }); - const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); - expect(queueCollection({ previewId: preview.id })).toEqual(result); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockClear(); - for (const players of [2, null]) { - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players); - await processDestructions(); - await processCreations(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('Waiting for removal'); - } - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); - await processDestructions(); - await processCreations(); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 2, done: 2, active: 0 }); - }); - - it('does not reuse a port after a removal is cancelled', async () => { - vi.stubEnv('SERVER_PORT_MAX', '28960'); - const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); - await processCreations(); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js'); - expect(cancelRemoval(getCollection(result.collectionId!)!.servers.find(server => server.action === 'destroy')!.id)).toBe(true); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockClear(); - await processCreations(); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getCollection(result.collectionId!)?.collection.failed).toBe(1); - expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('(cancelled)'); - }); - - it('allows independent launches to pass a failed removal without exceeding the startup limit', async () => { - const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); - await processCreations(); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]); - const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - const { komodoCreateDeployment, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoCreateDeployment).mockClear(); - vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Could not stop container' }); - await processDestructions(); - await processCreations(); - const creates = getCollection(result.collectionId!)!.servers.filter(server => server.action === 'create'); - expect(creates[0]).toMatchObject({ state: 'queued', stage: expect.stringContaining('(failed)') }); - expect(creates[1]).toMatchObject({ state: 'awaiting-health' }); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(getCollection(result.collectionId!)!.collection).toMatchObject({ failed: 1, queued: 1, verifying: 1 }); - }); - - it('rejects insufficient final capacity before cancelling any existing creation', async () => { - vi.stubEnv('SERVER_PORT_MAX', '28960'); - const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); - if (!old.ok) throw new Error(old.message); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - expect(preview.ready).toBe(false); - expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' }); - expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); - }); - - it('requires a new review when a pending removal is cancelled', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const old = await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); - if (!old.ok) throw new Error(old.message); - const removal = destroyServer(old.name); - if (!removal.ok) throw new Error(removal.message); - const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const preview = previewCollection(plan.id, 'exact'); - const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js'); - cancelRemoval(removal.deploymentId!); - expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/fleet changed/); - expect(previewCollection(plan.id, 'exact')).toMatchObject({ servers: [], removals: [], retainedServers: [old.name] }); - }); - - it('keeps T6 variants distinct and leaves a matching fleet untouched', async () => { - const { createServer } = await import('../../src/lib/server/fleet/create.js'); - await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town' }); - const plan = await blueprint([{ game: 't6', mapSlug: 'town', count: 1 }]); - const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); - const noOp = previewCollection(plan.id, 'exact'); - expect(noOp).toMatchObject({ retainedServers: ['t6_zombie_town_1'], removals: [], servers: [] }); - expect(queueCollection({ previewId: noOp.id })).toEqual({ ok: true, collectionId: null }); - await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm' }); - const preview = previewCollection(plan.id, 'exact'); - expect(preview.retainedServers).toEqual(['t6_zombie_town_1']); - expect(preview.removals.map(server => server.name)).toEqual(['t6_zombie_farm_1']); - }); - it('defaults old previews and collections to fill and validates the preview mode API', async () => { const { blueprint: plan } = await previewLineup(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); diff --git a/tests/unit/destruction-concurrency.test.ts b/tests/unit/destruction-concurrency.test.ts deleted file mode 100644 index 5f84366..0000000 --- a/tests/unit/destruction-concurrency.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { join } from 'node:path'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(), - komodoDeploy: vi.fn(), - komodoDestroyDeployment: vi.fn(), - komodoDeleteDeployment: vi.fn(), - komodoInspectHealthy: vi.fn(), - komodoEnsureTag: vi.fn(), - komodoUpdateTagsOnResource: vi.fn() -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() -})); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); - -function deferred() { - let resolve!: () => void; - let reject!: (error: Error) => void; - const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); - return { promise, resolve, reject }; -} - -function command() { - return { started: deferred(), response: deferred() }; -} - -const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`); -let fixture: Fixture; -let commands: Map; delete: ReturnType }>; -let passes: Promise[]; -let runningAtSubmission: number[]; - -beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); - fixture = await newFixture(servers); - for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', - KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value); - commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }])); - passes = []; - runningAtSubmission = []; - const { db } = await import('../../src/lib/server/db/index.js'); - for (const [index, server] of servers.entries()) { - db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir) - VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`) - .run(server, server, 28960 + index, join(fixture.t5, server)); - } - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValue(0); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - vi.mocked(komodoDestroyDeployment).mockImplementation(async server => { - runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'") - .get() as { n: number }).n); - const gate = commands.get(server)!.destroy; - gate.started.resolve(); - await gate.response.promise; - return { ok: true }; - }); - vi.mocked(komodoDeleteDeployment).mockImplementation(async server => { - const gate = commands.get(server)!.delete; - gate.started.resolve(); - await gate.response.promise; - return { ok: true }; - }); -}); - -afterEach(async () => { - for (const server of servers) complete(server); - await Promise.allSettled(passes); - await fixture.cleanup(); - vi.unstubAllEnvs(); - vi.restoreAllMocks(); -}); - -function complete(server: string) { - commands.get(server)!.destroy.response.resolve(); - commands.get(server)!.delete.response.resolve(); -} - -function track(pass: Promise) { - passes.push(pass); - return pass; -} - -async function queue(count = servers.length) { - const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); - return servers.slice(0, count).map(server => { - const result = destroyServer(server); - if (!result.ok) throw new Error(result.message); - return result.deploymentId!; - }); -} - -async function states() { - const { db } = await import('../../src/lib/server/db/index.js'); - return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid") - .all() as { state: string }[]).map(row => row.state); -} - -describe('destruction concurrency', () => { - it('defaults to one full removal at a time and refills the slot in the same pass', async () => { - await queue(2); - const { fleetSettings } = await import('../../src/lib/server/settings.js'); - const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - expect(fleetSettings().destroyConcurrency).toBe(1); - const pass = track(processDestructions()); - await commands.get(servers[0])!.destroy.started.promise; - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - expect(await states()).toEqual(['running', 'waiting']); - complete(servers[0]); - await commands.get(servers[1])!.destroy.started.promise; - expect(await states()).toEqual(['done', 'running']); - complete(servers[1]); - await pass; - expect(await states()).toEqual(['done', 'done']); - expect(runningAtSubmission).toEqual([1, 1]); - }); - - it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => { - await queue(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); - const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const pass = track(processDestructions()); - await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); - const overlapping = track(processDestructions()); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); - expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']); - commands.get(servers[0])!.destroy.response.resolve(); - await commands.get(servers[0])!.delete.started.promise; - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); - commands.get(servers[0])!.delete.response.resolve(); - await commands.get(servers[2])!.destroy.started.promise; - expect(await states()).toEqual(['done', 'running', 'running', 'waiting']); - complete(servers[1]); - await commands.get(servers[3])!.destroy.started.promise; - complete(servers[2]); complete(servers[3]); - await Promise.all([pass, overlapping]); - expect(await states()).toEqual(['done', 'done', 'done', 'done']); - expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers); - expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4); - expect(Math.max(...runningAtSubmission)).toBe(2); - }); - - it('lets active removals finish after lowering the limit and applies it to new admissions', async () => { - await queue(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); - const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const pass = track(processDestructions()); - await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); - saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 }); - complete(servers[0]); - await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting'])); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); - complete(servers[1]); - await commands.get(servers[2])!.destroy.started.promise; - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3); - expect(await states()).toEqual(['done', 'done', 'running', 'waiting']); - complete(servers[2]); - await commands.get(servers[3])!.destroy.started.promise; - complete(servers[3]); - await pass; - expect(runningAtSubmission.slice(2)).toEqual([1, 1]); - }); - - it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => { - await queue(3); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); - const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - if (failure === 'rejected') { - vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { - commands.get(servers[0])!.destroy.started.resolve(); - await commands.get(servers[0])!.destroy.response.promise; - return { ok: false, kind: 'http-error', message: 'Teardown rejected' }; - }); - } - const pass = track(processDestructions()); - await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); - if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost')); - else commands.get(servers[0])!.destroy.response.resolve(); - await commands.get(servers[2])!.destroy.started.promise; - expect(await states()).toEqual(['failed', 'running', 'running']); - complete(servers[1]); complete(servers[2]); - await pass; - expect(await states()).toEqual(['failed', 'done', 'done']); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3); - expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]); - }); - - it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => { - await queue(); - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); - const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null); - vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1] - ? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' }); - for (const server of servers) complete(server); - await track(processDestructions()); - expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']); - expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2)); - expect(rconGetPlayerCount).toHaveBeenCalledTimes(4); - }); -}); diff --git a/tests/unit/drift-kinds.test.ts b/tests/unit/drift-kinds.test.ts index 7307eb5..f6a3357 100644 --- a/tests/unit/drift-kinds.test.ts +++ b/tests/unit/drift-kinds.test.ts @@ -31,7 +31,7 @@ async function applyDriftDeployment(input: import('../../src/lib/server/fleet/dr beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_zombie_theater_1']); - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; // Ensure the three per-kind destinations exist under the fixture tree. await mkdir(join(fixture.t5, 't5_zombie_theater_1', 'runtime/plutonium/storage/t5/raw/scripts/sp'), { recursive: true }); diff --git a/tests/unit/fleet-monitor.test.ts b/tests/unit/fleet-monitor.test.ts index 375f768..63bac0f 100644 --- a/tests/unit/fleet-monitor.test.ts +++ b/tests/unit/fleet-monitor.test.ts @@ -37,7 +37,7 @@ async function setup() { describe('fleet event refreshes', () => { it('refreshes server history through one scoped request without replacing the fleet snapshot', async () => { const store = await import('../../src/lib/ui/stores/fleet.js'); - const integrations = { rcon: true, komodo: true, rconTimeoutMs: 2000, defaultRconHost: '' }; + const integrations = { rcon: true, rconTimeoutMs: 2000, defaultRconHost: '' }; const snapshot = { servers: [], operations: [], integrations, serverNow: Date.now() }; store.hydrateFleet(snapshot); const originalFleet = get(store.fleet); diff --git a/tests/unit/fleet-overview.test.ts b/tests/unit/fleet-overview.test.ts index c8d6265..9892241 100644 --- a/tests/unit/fleet-overview.test.ts +++ b/tests/unit/fleet-overview.test.ts @@ -54,7 +54,6 @@ describe('Fleet discovery and match freshness', () => { }); }); - describe('persisted Fleet operations', () => { let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); }); @@ -75,7 +74,7 @@ describe('persisted Fleet operations', () => { it('recovers waiting and recovering campaigns without needing a live event', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('one', 'promote', ?)").run(iso(-2 * 86_400_000)); handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players) VALUES ('one', 'server', 1, 'pending', 4)").run(); const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js'); @@ -109,7 +108,7 @@ describe('persisted Fleet operations', () => { it('surfaces orphaned recovery errors and beta gates', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); handle.prepare("INSERT INTO restart_rollout (server, requested_at, error) VALUES ('server', ?, 'RCON not answering')").run(iso()); const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js'); expect(fleetOperations(now)[0]).toMatchObject({ kind: 'recovery', state: 'attention', detail: 'RCON not answering' }); diff --git a/tests/unit/fleet-snapshot-scope.test.ts b/tests/unit/fleet-snapshot-scope.test.ts index 8984f44..2d44dba 100644 --- a/tests/unit/fleet-snapshot-scope.test.ts +++ b/tests/unit/fleet-snapshot-scope.test.ts @@ -10,11 +10,11 @@ beforeEach(async () => { fixture = await newFixture([]); const { db } = await import('../../src/lib/server/db/index.js'); db().transaction(() => { - const server = db().prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)'); + const server = db().prepare('INSERT INTO servers(name) VALUES (?)'); const operation = db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES(?,?,'restart',?,'Complete',?,?,'test')`); for (const name of ['one', 'two']) { - server.run(name, name); + server.run(name); for (let i = 0; i < 100; i++) operation.run(`${name}-${i}`, name, 'done', iso(-i), iso(-i)); operation.run(`${name}-old`, name, 'done', iso(-86_400_001), iso(-86_400_001)); operation.run(`${name}-failed`, name, 'failed', iso(), iso()); diff --git a/tests/unit/github-plugin-import.test.ts b/tests/unit/github-plugin-import.test.ts index a82393d..575d8a5 100644 --- a/tests/unit/github-plugin-import.test.ts +++ b/tests/unit/github-plugin-import.test.ts @@ -16,7 +16,7 @@ let fetchMock: ReturnType; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; fetchMock = vi.fn(async (input: string | URL) => String(input).startsWith('https://api.github.com/') ? Response.json(metadata()) : new Response(content)); diff --git a/tests/unit/health-observations.test.ts b/tests/unit/health-observations.test.ts index d691ec8..44ba4db 100644 --- a/tests/unit/health-observations.test.ts +++ b/tests/unit/health-observations.test.ts @@ -2,7 +2,6 @@ import { randomUUID } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn() })); const server = 'managed'; @@ -22,11 +21,12 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreA async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); - database.prepare('INSERT INTO servers(name,komodo_name,created_at,game_port) VALUES(?,?,?,28960)') - .run(server, server, new Date(now).toISOString()); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + database.prepare('INSERT INTO servers(name,created_at,game_port) VALUES(?,?,28960)') + .run(server, new Date(now).toISOString()); + vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const inspect = vi.mocked(komodoInspectHealthy).mockResolvedValue(healthy); + const inspect = vi.mocked(nativeHealth).mockResolvedValue(healthy); const rcon = vi.mocked(rconStatusResult).mockResolvedValue(playing); return { database, inspect, rcon, ...await import('../../src/lib/server/fleet/health-observations.js') }; } @@ -228,37 +228,6 @@ describe('shared monitoring observations', () => { expect(api.inspect).toHaveBeenCalledTimes(2); }); - it('invalidates integration changes and accepts legacy port discovery by the successful RCON probe', async () => { - const api = await setup(); - await api.inspectForMonitoring(server); - api.database.prepare("UPDATE servers SET komodo_name='replacement' WHERE name=?").run(server); - await api.inspectForMonitoring(server); - expect(api.inspect).toHaveBeenCalledTimes(2); - api.database.prepare('UPDATE servers SET game_port=NULL WHERE name=?').run(server); - api.rcon.mockImplementationOnce(async () => { - api.database.prepare('UPDATE servers SET game_port=28960 WHERE name=?').run(server); - return playing; - }); - expect((await api.rconForMonitoring(server)).result).toEqual(playing); - await api.rconForMonitoring(server); - expect(api.rcon).toHaveBeenCalledTimes(1); - api.database.prepare('UPDATE servers SET game_port=28961 WHERE name=?').run(server); - await api.rconForMonitoring(server); - expect(api.rcon).toHaveBeenCalledTimes(2); - }); - - it.each(['inspect', 'rcon'] as const)('does not reuse %s after a newer manual observation', async kind => { - const api = await setup(); - const call = kind === 'inspect' ? api.inspectForMonitoring : api.rconForMonitoring; - const request = kind === 'inspect' ? api.inspect : api.rcon; - const field = kind === 'inspect' ? 'container_checked_at' : 'last_checked_at'; - await call(server); - vi.setSystemTime(now + 1000); - api.database.prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(new Date(now + 1000).toISOString(), server); - await call(server); - expect(request).toHaveBeenCalledTimes(2); - }); - it('rejects a previous generation completing after replacement without discarding the new generation cache', async () => { const api = await setup(); const { instanceId } = await native(api); diff --git a/tests/unit/host-instances.test.ts b/tests/unit/host-instances.test.ts index 33082f4..aa34f78 100644 --- a/tests/unit/host-instances.test.ts +++ b/tests/unit/host-instances.test.ts @@ -16,10 +16,10 @@ describe('host detail instances', () => { db().prepare(`INSERT INTO native_instances (id, server, host_id, generation, library_id, image, port, created_at, retired_at) VALUES (?, ?, ?, 1, 'library', 'runtime:test', 28960, ?, ?)`) .run(name, name, hostId, now, retired ? now : null); - db().prepare(`INSERT INTO servers (name, komodo_name, + db().prepare(`INSERT INTO servers (name, container_state, container_checked_at, last_seen_at, last_players, destroyed_at) - VALUES (?, ?, 'running', ?, ?, 2, ?)`) - .run(name, name, now, now, destroyed ? now : null); + VALUES (?, 'running', ?, ?, 2, ?)`) + .run(name, now, now, destroyed ? now : null); }; add('running', host.id); add('unavailable', host.id); diff --git a/tests/unit/host-status-batch.test.ts b/tests/unit/host-status-batch.test.ts index 463e08c..b96480f 100644 --- a/tests/unit/host-status-batch.test.ts +++ b/tests/unit/host-status-batch.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() })); let fixture: Fixture; let at: number; beforeEach(async () => { @@ -15,8 +14,9 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function setup(count = 5, hostCount = 1) { const hosts = await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); + vi.mocked(nativeHealth).mockResolvedValue({ ok: true, state: 'running' }); const hostIds: string[] = []; for (let h = 0; h < hostCount; h++) { const host = hosts.createHost({ name: `Host ${h}` }); hostIds.push(host.id); @@ -26,7 +26,7 @@ async function setup(count = 5, hostCount = 1) { hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); for (let i = 0; i < count; i++) { const name = `server-${h}-${i}`; - db().prepare('INSERT INTO servers(name,komodo_name,created_at) VALUES(?,?,?)').run(name, '', new Date(at - 1000).toISOString()); + db().prepare('INSERT INTO servers(name,created_at) VALUES(?,?)').run(name, new Date(at - 1000).toISOString()); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',?,?)`).run(`instance-${h}-${i}`, name, host.id, 28960 + i, new Date(at - 1000).toISOString()); } @@ -41,7 +41,7 @@ async function setup(count = 5, hostCount = 1) { ? { ok: true, results: targets.map(target => ({ ...target, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } })) } : { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } }); }); - return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(komodoInspectHealthy), hostIds, operations, complete }; + return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(nativeHealth), hostIds, operations, complete }; } describe('host status observations', () => { diff --git a/tests/unit/komodo-health.test.ts b/tests/unit/komodo-health.test.ts deleted file mode 100644 index f8d661b..0000000 --- a/tests/unit/komodo-health.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { komodoInspectHealthy } from '../../src/lib/server/integrations/komodo.js'; - -vi.mock('../../src/lib/server/config.js', () => ({ - config: { - komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' }, - httpTimeoutMs: 1000 - }, - komodoConfigured: () => true -})); -vi.mock('../../src/lib/server/fleet/servers.js', () => ({ - getServer: () => ({ komodo_type: 'deployment', komodo_name: 'saved-resource' }) -})); - -const fetchMock = vi.fn(); - -beforeEach(() => { - fetchMock.mockReset(); - vi.stubGlobal('fetch', fetchMock); -}); -afterEach(() => { - vi.unstubAllGlobals(); - vi.useRealTimers(); -}); - -describe('Komodo health endpoint fallback', () => { - it('limits an unreachable probe to one timeout and preserves its diagnosis', async () => { - vi.useFakeTimers(); - fetchMock.mockImplementation((_url, options) => new Promise((_resolve, reject) => { - options?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))); - })); - - const result = komodoInspectHealthy('server'); - await vi.advanceTimersByTimeAsync(1000); - - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(await result).toMatchObject({ - ok: false, - reason: 'timed out after 1000ms dialing http://komodo.test/read/GetDeploymentContainer' - }); - }); - - it('preserves the network cause without probing alternate endpoints', async () => { - fetchMock.mockRejectedValue(new TypeError('fetch failed', { - cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' }) - })); - - expect(await komodoInspectHealthy('server')).toMatchObject({ - ok: false, - reason: expect.stringContaining('ECONNREFUSED: connection refused') - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it.each([400, 401, 403, 429, 500, 502, 503])('stops on HTTP %i without hiding the response', async status => { - fetchMock.mockResolvedValue(new Response('request failed', { status })); - - expect(await komodoInspectHealthy('server')).toEqual({ - ok: false, reason: `HTTP ${status}`, snippet: 'request failed' - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it.each([ - [404, 'Not Found'], - [400, 'unknown variant `GetDeploymentContainer`'], - [405, 'unsupported method'], - [500, 'unknown method'], - [501, 'method not found'] - ])('tries a compatible endpoint after HTTP %i: %s', async (status, message) => { - fetchMock - .mockResolvedValueOnce(new Response(message as string, { status: status as number })) - .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } })); - - expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' }); - expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ - 'http://komodo.test/read/GetDeploymentContainer', - 'http://komodo.test/read/InspectDeploymentContainer' - ]); - }); - - it('continues past successful responses with an unknown shape', async () => { - fetchMock - .mockResolvedValueOnce(Response.json({ config: { image: 'server' } })) - .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } })); - - expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' }); - expect(fetchMock).toHaveBeenCalledTimes(2); - }); - - it('does not overwrite an unhealthy container state with a later response', async () => { - fetchMock.mockResolvedValueOnce(Response.json({ State: { Status: 'exited' } })); - - expect(await komodoInspectHealthy('server')).toMatchObject({ ok: false, state: 'exited', reason: 'container state: exited' }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it('preserves malformed response diagnostics', async () => { - fetchMock.mockResolvedValueOnce(new Response('not json')); - - expect(await komodoInspectHealthy('server')).toMatchObject({ - ok: false, reason: expect.stringContaining('unparseable json:'), snippet: 'not json' - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); -}); diff --git a/tests/unit/komodo-runtime.test.ts b/tests/unit/komodo-runtime.test.ts deleted file mode 100644 index 70a1288..0000000 --- a/tests/unit/komodo-runtime.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { afterEach, beforeEach, expect, it, vi } from 'vitest'; - -vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn() })); -beforeEach(() => { - vi.resetModules(); vi.clearAllMocks(); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: true }))); -}); -afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); - -it.each(['start', 'stop'] as const)('sends %s to the saved resource for each Komodo type', async action => { - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js'); - const verb = action === 'start' ? 'Start' : 'Stop'; - for (const [type, suffix, body] of [ - ['deployment', 'Deployment', { deployment: 'saved-resource' }], - ['stack', 'Stack', { stack: 'saved-resource' }], - ['container', 'Container', { server: 'host', container: 'saved-resource' }] - ] as const) { - vi.mocked(getServer).mockReturnValue({ komodo_type: type, komodo_name: 'saved-resource', komodo_server: 'host' } as ReturnType); - expect(await komodoRuntimeAction('ec-server-id', action)).toEqual({ ok: true }); - expect(fetch).toHaveBeenLastCalledWith(`http://komodo.test/execute/${verb}${suffix}`, expect.objectContaining({ method: 'POST', body: JSON.stringify(body) })); - } -}); - -it('rejects missing resource identity and surfaces a rejected stop response', async () => { - const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(getServer).mockReturnValue(undefined); - expect(await komodoRuntimeAction('missing', 'stop')).toMatchObject({ ok: false, kind: 'no-name' }); - expect(fetch).not.toHaveBeenCalled(); - vi.mocked(getServer).mockReturnValue({ komodo_type: 'deployment', komodo_name: 'saved' } as ReturnType); - vi.mocked(fetch).mockResolvedValueOnce(Response.json({ success: false, message: 'rejected' })); - expect(await komodoRuntimeAction('server', 'stop')).toMatchObject({ ok: false, message: 'rejected' }); -}); diff --git a/tests/unit/komodo-tags.test.ts b/tests/unit/komodo-tags.test.ts deleted file mode 100644 index 66a2d8b..0000000 --- a/tests/unit/komodo-tags.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { komodoEnsureTag, komodoUpdateTagsOnResource } from '../../src/lib/server/integrations/komodo.js'; - -vi.mock('../../src/lib/server/config.js', () => ({ - config: { - komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' }, - httpTimeoutMs: 1000 - }, - komodoConfigured: () => true -})); - -const fetchMock = vi.fn(); -const gameId = '6aa1fb6bc27e735cc0767f5a'; -const mapId = '6aabe8567a227ba7d9aad3e9'; -const tags = [ - { _id: { $oid: gameId }, name: 't5', owner: '6aa1e0ba91f87b7712e8caf8', color: 'Slate' }, - { _id: { $oid: mapId }, name: 'zombie_cod5_sumpf', owner: '6aa1e0ba91f87b7712e8caf8', color: 'Slate' } -]; - -beforeEach(() => { - fetchMock.mockReset(); - vi.stubGlobal('fetch', fetchMock); - vi.spyOn(console, 'warn').mockImplementation(() => {}); -}); - -afterEach(() => { - vi.unstubAllGlobals(); - vi.restoreAllMocks(); -}); - -describe('Komodo tags', () => { - it('reuses the tags from the logged Mongo response and sends string IDs to the deployment', async () => { - fetchMock - .mockResolvedValueOnce(Response.json(tags)) - .mockResolvedValueOnce(Response.json(tags)) - .mockResolvedValueOnce(Response.json({})); - - const game = await komodoEnsureTag('t5'); - const map = await komodoEnsureTag('zombie_cod5_sumpf'); - expect(game).toEqual({ ok: true, id: gameId }); - expect(map).toEqual({ ok: true, id: mapId }); - if (!game.ok || !map.ok) throw new Error('Expected both tags to resolve'); - expect(await komodoUpdateTagsOnResource('Deployment', 't5-sumpf-1', [game.id, map.id])) - .toEqual({ ok: true }); - expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ - 'http://komodo.test/read/ListTags', - 'http://komodo.test/read/ListTags', - 'http://komodo.test/write/UpdateResourceMeta' - ]); - expect(JSON.parse(fetchMock.mock.calls[2][1]!.body as string)).toEqual({ - target: { type: 'Deployment', id: 't5-sumpf-1' }, tags: [gameId, mapId] - }); - expect(console.warn).not.toHaveBeenCalled(); - }); - - it.each([{ id: gameId }, { _id: gameId }])('preserves string ID support: %j', async (id) => { - fetchMock.mockResolvedValueOnce(Response.json([{ ...id, name: 't5' }])); - expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId }); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it('reads an Extended JSON ID from CreateTag without another lookup', async () => { - fetchMock - .mockResolvedValueOnce(Response.json([])) - .mockResolvedValueOnce(Response.json([])) - .mockResolvedValueOnce(Response.json(tags[0])); - expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId }); - expect(fetchMock).toHaveBeenCalledTimes(3); - expect(fetchMock.mock.calls[2][0]).toBe('http://komodo.test/write/CreateTag'); - }); - - it('recovers when another caller creates the tag after the initial lookup', async () => { - fetchMock - .mockResolvedValueOnce(Response.json([])) - .mockResolvedValueOnce(Response.json([])) - .mockResolvedValueOnce(Response.json({ error: 'E11000 duplicate key error' }, { status: 500 })) - .mockResolvedValueOnce(Response.json(tags)); - expect(await komodoEnsureTag('t5')).toEqual({ ok: true, id: gameId }); - expect(fetchMock).toHaveBeenCalledTimes(4); - }); - - it.each([null, {}, { $oid: '' }, { $oid: 123 }])('rejects malformed Mongo IDs: %j', async (_id) => { - fetchMock - .mockResolvedValueOnce(Response.json([{ name: 't5', _id }])) - .mockResolvedValueOnce(Response.json({ error: 'create failed' }, { status: 400 })); - expect(await komodoEnsureTag('t5')).toMatchObject({ ok: false }); - }); -}); diff --git a/tests/unit/komodo-write.test.ts b/tests/unit/komodo-write.test.ts deleted file mode 100644 index 036dac5..0000000 --- a/tests/unit/komodo-write.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -beforeEach(() => { - vi.resetModules(); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test-key'); - vi.stubEnv('KOMODO_API_SECRET', 'test-secret'); -}); -afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); - -describe('Komodo startup response classification', () => { - it.each([403, 502])('preserves HTTP %i so callers can distinguish rejection from uncertain startup', async httpStatus => { - vi.stubGlobal('fetch', vi.fn(async () => new Response('Deployment response failed', { status: httpStatus }))); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus }); - }); - - it('identifies an explicit success=false response as a confirmed HTTP 200 failure', async () => { - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: false, message: 'Image rejected' }))); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus: 200, message: 'Image rejected' }); - }); - - it('returns a structured unreachable result when the startup response times out', async () => { - vi.stubGlobal('fetch', vi.fn(async () => { throw new DOMException('Request aborted', 'AbortError'); })); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'unreachable', message: expect.stringContaining('timed out') }); - }); -}); diff --git a/tests/unit/komodo.test.ts b/tests/unit/komodo.test.ts deleted file mode 100644 index dc746bb..0000000 --- a/tests/unit/komodo.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js'; - -describe('container readiness', () => { - it.each([ - ['Up 35 seconds (health: starting)', 'starting'], - ['Up about a minute (healthy)', 'running'], - ['Up 5 minutes (unhealthy)', 'unhealthy'], - ['Up 2 hours', 'running'], - ['Exited (1) 5 seconds ago', 'exited'] - ])('reads nested Komodo container status: %s', (status, expected) => { - expect(extractContainerState({ state: 'Running', container: { state: 'running', status } })).toBe(expected); - }); - it('reads lowercase inspect health and does not trust missing containers or configuration', () => { - expect(extractContainerState({ state: { status: 'running', running: true, health: { status: 'starting' } } })).toBe('starting'); - expect(extractContainerState({ state: 'Running', container: null })).toBe('unknown'); - expect(extractContainerState({ config: { state: 'running' }, status: 'ok' })).toBeNull(); - expect(extractContainerState({ state: 'Exited', container: { status: 'Up 1 minute (healthy)' } })).toBe('exited'); - }); - it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => { - expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health); - expect(extractContainerState({ state: 'running', health })).toBe(health); - expect(extractContainerState({ state: 'running', health: { status: health } })).toBe(health); - }); - it('accepts healthy running containers and preserves stopped states', () => { - expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running'); - expect(extractContainerState({ State: { Status: 'exited', Running: false, Health: { Status: 'healthy' } } })).toBe('exited'); - }); -}); - -describe('extractUdpPort', () => { - it('returns the published port from docker-compose-style object entries', () => { - const payload = { - state: { - container: { - ports: [ - { protocol: 'tcp', published: 3000, target: 3000 }, - { protocol: 'udp', published: 4976, target: 28960 } - ] - } - } - }; - expect(extractUdpPort(payload)).toBe(4976); - }); - - it('parses docker-run-style "host:container/proto" strings', () => { - const payload = { ports: ['3074:3074/tcp', '4977:28960/udp'] }; - expect(extractUdpPort(payload)).toBe(4977); - }); - - it('picks the UDP entry even when TCP appears first', () => { - const payload = { - deeply: { - nested: { - ports: [ - { Protocol: 'tcp', PublishedPort: 80 }, - { Protocol: 'udp', PublishedPort: 5555 } - ] - } - } - }; - expect(extractUdpPort(payload)).toBe(5555); - }); - - it('returns null when no UDP mapping exists', () => { - const payload = { ports: [{ protocol: 'tcp', published: 3000 }] }; - expect(extractUdpPort(payload)).toBeNull(); - }); - - it('survives cyclic references', () => { - const cyc: Record = {}; - cyc.self = cyc; - expect(extractUdpPort(cyc)).toBeNull(); - }); - - it('recognizes HostPort / hostPort / host_port on object entries', () => { - expect(extractUdpPort({ ports: [{ protocol: 'udp', HostPort: 28962 }] })).toBe(28962); - expect(extractUdpPort({ ports: [{ protocol: 'udp', hostPort: 28963 }] })).toBe(28963); - expect(extractUdpPort({ ports: [{ protocol: 'udp', host_port: 28964 }] })).toBe(28964); - }); - - it('recognizes `type` alongside `protocol` as the protocol field', () => { - expect(extractUdpPort({ ports: [{ type: 'UDP', published: 28965 }] })).toBe(28965); - }); - - it('parses host-network "container/udp" strings', () => { - expect(extractUdpPort({ ports: ['28966/udp'] })).toBe(28966); - }); - - it('handles docker-inspect container-keyed PortBindings shape', () => { - const payload = { - NetworkSettings: { - Ports: { - '28960/udp': [{ HostIp: '0.0.0.0', HostPort: '28967' }], - '3074/tcp': [{ HostIp: '0.0.0.0', HostPort: '3074' }] - } - } - }; - expect(extractUdpPort(payload)).toBe(28967); - }); - - it('prefers `ports`-keyed arrays over unrelated arrays discovered deeper', () => { - // The unrelated array happens to contain a matching entry — the walker - // must still prefer the real `ports` list. - const payload = { - labels: [{ protocol: 'udp', published: 9999 }], - state: { container: { ports: [{ protocol: 'udp', published: 28968 }] } } - }; - expect(extractUdpPort(payload)).toBe(28968); - }); - - it('splits Komodo`s newline-delimited string form of `config.ports`', () => { - // Verbatim shape from a real Komodo GetDeployment response — the - // container publishes UDP 28963 and TCP 28963 as a single string. - const payload = { - name: 't5-nacht-1', - config: { - image: { type: 'Image', params: { image: 'ghcr.io/ayymoss/plutainer:latest' } }, - network: 'komodo_default', - ports: '28963:28963/udp\n28963:28963/tcp\n' - } - }; - expect(extractUdpPort(payload)).toBe(28963); - }); - - it('handles a single-line string-form ports value', () => { - expect(extractUdpPort({ config: { ports: '28969:28960/udp' } })).toBe(28969); - }); - - it('handles a host-network string-form ports value (no host prefix)', () => { - expect(extractUdpPort({ config: { ports: '28970/udp\n' } })).toBe(28970); - }); - - it('ignores TCP-only string-form ports entries', () => { - expect(extractUdpPort({ config: { ports: '80:80/tcp\n443:443/tcp' } })).toBeNull(); - }); -}); - -describe('komodoResolvePortDetailed', () => { - it('is importable and has the discriminated-union shape at the type level', async () => { - // Runtime import shim just to prove it's exported. Behavior is - // covered by the fixture-based extractUdpPort tests above plus manual - // end-to-end verification against a live Komodo. - const mod = await import('../../src/lib/server/integrations/komodo.js'); - expect(typeof mod.komodoResolvePortDetailed).toBe('function'); - expect(typeof mod.komodoResolveGamePort).toBe('function'); - }); -}); diff --git a/tests/unit/launch-recipes.test.ts b/tests/unit/launch-recipes.test.ts index 30ce810..d15ce7b 100644 --- a/tests/unit/launch-recipes.test.ts +++ b/tests/unit/launch-recipes.test.ts @@ -1,17 +1,9 @@ +import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoCreateDeployment: vi.fn(async () => ({ ok: true })), - komodoDeploy: vi.fn(async () => ({ ok: true })), - komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), - komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), - komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })), - komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), - komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })), @@ -27,11 +19,9 @@ const input = { ...selection, gameMode: 'zombie' as const, plutoKey: 'private-pl beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); + await prepareNativeHost(); vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'private-api-key'); - vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret'); - vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); + vi.stubEnv('RCON_PASSWORD', 'private-rcon-password'); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); @@ -46,42 +36,11 @@ async function seedArtifacts() { } describe('launch recipes and previews', () => { - it('keeps detail polling independent of retained plugin bytes after launching', async () => { - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const plugin = createScript({ name: 'Large utility plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5', - targetRelPath: 'large.dll', content: Buffer.alloc(8 * 1024 ** 2, 42) }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(plugin.version.id, plugin.script.id); - const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const queued = queueCreateServer(input); - if (!queued.ok) throw new Error(queued.message); - const queuedSummary = getDeployment(queued.deploymentId)!.recipe!; - await processCreations(); - const full = recipeForDeployment(queued.deploymentId)!.recipe; - const expected = summarizeRecipe(full); - expect(expected.fingerprint).not.toBe(queuedSummary.fingerprint); - expect(full.files.find(file => file.path === 'large.dll')?.content).toBe(plugin.version.content.toString('base64')); - closeDb(); - const parse = JSON.parse; - const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => { - if (value.length > 1024 * 1024) throw new Error('Detail polling parsed plugin bytes.'); - return parse(value, reviver); - }); - try { - for (let read = 0; read < 3; read++) expect(getDeployment(queued.deploymentId)?.recipe).toEqual(expected); - const row = db().prepare('SELECT length(recipe_summary) AS bytes FROM server_launch_recipes WHERE deployment_id = ?') - .get(queued.deploymentId) as { bytes: number }; - expect(row.bytes).toBeLessThan(10_000); - } finally { parsing.mockRestore(); } - }); - it('previews without reserving capacity, saving keys, exposing bytes, or calling Komodo', async () => { + it('previews without reserving capacity, saving keys, exposing bytes, or submitting host commands', async () => { await seedArtifacts(); const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const first = previewCreation(selection); const second = previewCreation(selection); expect(first).toMatchObject({ name: 't5_zombie_theater_1', port: 28960, hasSavedKey: false, ready: true }); @@ -91,68 +50,9 @@ describe('launch recipes and previews', () => { expect(first.recipe.files[0]).not.toHaveProperty('content'); expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(JSON.stringify(first)).not.toContain('private-'); }); - it('uses promotions made after review and queueing, including plugins and composed configuration', async () => { - const { baseline, script } = await seedArtifacts(); - const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const plugin = createScript({ name: 'Game utilities', kind: 'plugin', scopeType: 'game', scopeValue: 't5', targetRelPath: 'utilities.dll', content: Buffer.from('old plugin') }); - const overlay = createScript({ name: 'Kino settings', kind: 'config', scopeType: 'map', scopeValue: 'zombie_theater', content: Buffer.from('set g_speed "180"\n') }); - for (const item of [plugin, overlay]) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(item.version.id, item.script.id); - const preview = previewCreation(selection); - const newScript = uploadNewVersion(script.script.id, Buffer.from('main(){ new_perks(); }')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newScript.id, script.script.id); - const newOverlay = uploadNewVersion(overlay.script.id, Buffer.from('set g_speed "200"\n')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newOverlay.id, overlay.script.id); - uploadNewVersion(script.script.id, Buffer.from('main(){ unpromoted_draft(); }')); - const queued = queueCreateServer({ ...input, previewId: preview.id }); - if (!queued.ok) throw new Error(queued.message); - expect(recipeForDeployment(queued.deploymentId)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newScript.id })); - const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newBaseline.id, baseline.script.id); - const newPlugin = uploadNewVersion(plugin.script.id, Buffer.from('new plugin')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newPlugin.id, plugin.script.id); - uploadNewVersion(plugin.script.id, Buffer.from('unpromoted plugin')); - closeDb(); - await processCreations(); - expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('main(){ new_perks(); }'); - const deployedConfig = await readFile(join(fixture.t5, queued.name, 'configs/dedicated_sp.cfg'), 'utf8'); - expect(deployedConfig).toContain('set sv_voice "0"'); - expect(deployedConfig).toContain('set g_speed "200"'); - expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/plugins/utilities.dll'), 'utf8')).toBe('new plugin'); - expect(getDeployment(queued.deploymentId)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'utilities.dll', versionId: newPlugin.id })); - expect(getServerScript(queued.name, 'perks.gsc')).toMatchObject({ script_version_id: newScript.id, sha256: newScript.sha256 }); - expect(getServerScript(queued.name, 'utilities.dll')).toMatchObject({ script_version_id: newPlugin.id, sha256: newPlugin.sha256 }); - const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg'); - if (recordedConfig?.content === undefined) throw new Error('Attempted launch must retain the configuration bytes.'); - expect(Buffer.from(recordedConfig.content, 'base64').toString()).toBe(deployedConfig); - expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint); - expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); - await processCreations(); - expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'done' }); - expect(JSON.stringify(getDeployment(queued.deploymentId))).not.toContain('private-'); - }); - - it('uses the latest promotion at provisioning for API clients without a preview', async () => { - const { script } = await seedArtifacts(); - const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const queued = queueCreateServer(input); - if (!queued.ok) throw new Error(queued.message); - const changed = uploadNewVersion(script.script.id, Buffer.from('changed')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(changed.id, script.script.id); - await processCreations(); - expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('changed'); - }); - it('returns the same operation for repeated submission, even after expiry and restart', async () => { const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); @@ -167,31 +67,6 @@ describe('launch recipes and previews', () => { expect(getServerKey(preview.name!)).toBe('private-pluto-key'); }); - it('rejects stale slots and ports before saving a key or queueing a second server', async () => { - const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const preview = previewCreation(selection); - expect(queueCreateServer({ ...input, mapSlug: 'moon' }).ok).toBe(true); - expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'no-slot' }); - expect(getServerKey(preview.name!)).toBeUndefined(); - const fresh = previewCreation(selection); - expect(fresh.port).toBe(28961); - expect(queueCreateServer({ ...input, previewId: fresh.id }).ok).toBe(true); - expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'no-slot' }); - expect(getServerKey('t5_zombie_theater_2')).toBeUndefined(); - }); - - it('does not provision an already accepted preview again through internal callers', async () => { - const { previewCreation, createServer } = await import('../../src/lib/server/fleet/create.js'); - const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const preview = previewCreation(selection); - const first = await createServer({ ...input, previewId: preview.id }); - expect(first.ok).toBe(true); - expect(await createServer({ ...input, previewId: preview.id })).toEqual(first); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - }); - it('carries preview identity through the public creation API and rejects malformed requests', async () => { const { POST: preview } = await import('../../src/routes/api/deployments/preview/+server.js'); const { POST: launch } = await import('../../src/routes/api/servers/+server.js'); @@ -218,93 +93,6 @@ describe('launch recipes and previews', () => { expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]); }); - it('reports configuration blockers while still allowing recipes to be saved', async () => { - vi.stubEnv('KOMODO_TARGET_SERVER', ''); vi.stubEnv('RCON_PASSWORD', ''); - const { previewCreation, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const preview = previewCreation(selection); - expect(preview.ready).toBe(false); - expect(preview.checks.filter(c => c.state === 'blocked')).toHaveLength(2); - expect(saveBlueprint({ name: 'For later', previewId: preview.id }).name).toBe('For later'); - expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, kind: 'unconfigured' }); - }); - - it('resolves current file eligibility after a blueprint review and queued launch', async () => { - const { script } = await seedArtifacts(); - const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { saveBlueprint, deleteBlueprint, listBlueprints, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const removed = createScript({ name: 'Removed script', targetRelPath: 'removed.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('removed') }); - const unpromoted = createScript({ name: 'Unpromoted script', targetRelPath: 'unpromoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('unpromoted') }); - for (const item of [removed, unpromoted]) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(item.version.id, item.script.id); - const blueprint = saveBlueprint({ name: 'Classic night', description: 'Kino lineup', previewId: previewCreation(selection).id }); - const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string }; - expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries: [{ count: 1, ...selection }] }); - closeDb(); - expect(listBlueprints()[0]).toEqual(blueprint); - const preview = previewCreation({ blueprintId: blueprint.id }); - deleteBlueprint(blueprint.id); - const queued = queueCreateServer({ ...input, previewId: preview.id }); - if (!queued.ok) throw new Error(queued.message); - db().prepare('UPDATE scripts SET current_version_id = NULL WHERE id = ?').run(removed.script.id); - db().prepare('DELETE FROM scripts WHERE id = ?').run(removed.script.id); - db().prepare('UPDATE scripts SET current_version_id = NULL WHERE id = ?').run(unpromoted.script.id); - setScope({ scriptId: script.script.id, scopeType: 'map', scopeValue: 'zombie_moon' }); - const added = createScript({ name: 'New telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'map', scopeValue: 'zombie_theater', content: Buffer.from('newly promoted telemetry') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(added.version.id, added.script.id); - createScript({ name: 'Draft telemetry', targetRelPath: 'draft.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('never promoted') }); - await processCreations(); - for (const file of ['perks.gsc', 'removed.gsc', 'unpromoted.gsc', 'draft.gsc']) { - await expect(readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp', file))).rejects.toMatchObject({ code: 'ENOENT' }); - } - expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('newly promoted telemetry'); - expect(recipeForDeployment(queued.deploymentId)?.blueprintName).toBe('Classic night'); - expect(recipeForDeployment(queued.deploymentId)?.recipe.files.filter(file => file.kind === 'script')).toEqual([ - expect.objectContaining({ path: 'telemetry.gsc', versionId: added.version.id }) - ]); - }); - - it('launches again with current promotions while preserving the completed deployment record', async () => { - const { script } = await seedArtifacts(); - const { previewCreation, createServer, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const original = await createServer(input); - if (!original.ok) throw new Error(original.message); - const historicalRecipe = recipeForDeployment(original.deploymentId); - const promoted = uploadNewVersion(script.script.id, Buffer.from('current telemetry')); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(promoted.id, script.script.id); - const replay = previewCreation({ deploymentId: original.deploymentId }); - expect(replay).toMatchObject({ name: 't5_zombie_theater_2', port: 28961, hasSavedKey: false }); - expect(replay.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: promoted.id })); - const blueprint = saveBlueprint({ name: 'Restored', deploymentId: original.deploymentId }); - expect(blueprint.entries).toEqual([{ count: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino Der Toten' }]); - const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string }; - expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries: [{ count: 1, ...selection }] }); - expect(previewCreation({ blueprintId: blueprint.id }).recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: promoted.id })); - const queued = queueCreateServer({ ...input, previewId: replay.id }); - if (!queued.ok) throw new Error(queued.message); - await processCreations(); - expect(await readFile(join(fixture.t5, queued.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('current telemetry'); - expect(recipeForDeployment(original.deploymentId)).toEqual(historicalRecipe); - expect(await readFile(join(fixture.t5, original.name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('main(){ old_perks(); }'); - }); - - it('preserves the exact T6 map variant in a blueprint', async () => { - const { previewCreation, queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); - const preview = previewCreation({ game: 't6', mapSlug: 'town' }); - const blueprint = saveBlueprint({ name: 'Town survival', previewId: preview.id }); - const saved = previewCreation({ blueprintId: blueprint.id }); - expect(saved.recipe).toMatchObject({ game: 't6', mapSlug: 'town', expectedMap: 'zm_transit', configFile: 'dedicated_zm.cfg' }); - const queued = queueCreateServer({ ...input, game: 't6', mapSlug: 'town', previewId: saved.id }); - if (!queued.ok) throw new Error(queued.message); - await processCreations(); - expect(await readFile(join(fixture.t6, queued.name, 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); - }); - it('validates blueprint names, sources, and case-insensitive duplicates', async () => { const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); @@ -316,38 +104,3 @@ describe('launch recipes and previews', () => { expect(() => saveBlueprint({ name: ' classic night ', previewId })).toThrow(/already exists/); }); }); - -describe('resuming verification', () => { - it('resumes an interrupted creation without creating or restarting its container again', async () => { - const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { interruptDeployments, getDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const created = await createServer(input); - if (!created.ok) throw new Error(created.message); - db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(created.deploymentId); - interruptDeployments(); - expect(getDeployment(created.deploymentId)?.canResumeVerification).toBe(true); - expect(resumeVerification(created.deploymentId)).toBe(true); - expect(resumeVerification(created.deploymentId)).toBe(false); - await processCreations(); - expect(getDeployment(created.deploymentId)?.deployment).toMatchObject({ state: 'done', error: null }); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - }); - - it('refuses to observe a replacement server or interfere with a later destruction', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { finishDeployment, resumeVerification, getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const created = await createServer(input); - if (!created.ok) throw new Error(created.message); - finishDeployment(created.deploymentId, { ok: false, message: 'Interrupted' }); - db().prepare('UPDATE servers SET game_port = game_port + 1 WHERE name = ?').run(created.name); - expect(resumeVerification(created.deploymentId)).toBe(false); - db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(created.port, created.name); - expect(getDeployment(created.deploymentId)?.canResumeVerification).toBe(true); - destroyServer(created.name); - expect(resumeVerification(created.deploymentId)).toBe(false); - }); -}); diff --git a/tests/unit/leaderboard-records.test.ts b/tests/unit/leaderboard-records.test.ts index b603314..d14d9f3 100644 --- a/tests/unit/leaderboard-records.test.ts +++ b/tests/unit/leaderboard-records.test.ts @@ -122,8 +122,8 @@ it('migrates legacy participation as unknown before snapshot backfill and preser seed(handle, 1, { status: 'active' }); handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN first_observed_round'); handle.exec('ALTER TABLE leaderboard_match_players DROP COLUMN present_at_start'); - handle.prepare(`INSERT INTO servers (name, komodo_name, game, map) - VALUES ('t5_zombie_moon_1', 'moon', 't5', 'zombie_moon')`).run(); + handle.prepare(`INSERT INTO servers (name, game, map) + VALUES ('t5_zombie_moon_1', 't5', 'zombie_moon')`).run(); const saved: GameTelemetry = { event: 'game_end', server: 't5_zombie_moon_1', map: 'zombie_moon', round: 8, players: [player('202', 80)], receivedAt: timestamp(2) }; handle.prepare('INSERT INTO server_telemetry (server, round, players, received_at, payload) VALUES (?, ?, ?, ?, ?)') diff --git a/tests/unit/map-config-flow.test.ts b/tests/unit/map-config-flow.test.ts index 4b608db..0afc841 100644 --- a/tests/unit/map-config-flow.test.ts +++ b/tests/unit/map-config-flow.test.ts @@ -11,7 +11,7 @@ const cfgPath = (server: string) => join(fixture.t5, server, 'configs', 'dedicat beforeEach(async () => { vi.resetModules(); fixture = await newFixture(servers); - vi.stubEnv('KOMODO_ADDRESS', ''); + vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); diff --git a/tests/unit/match-detail.test.ts b/tests/unit/match-detail.test.ts index d384702..a7eb984 100644 --- a/tests/unit/match-detail.test.ts +++ b/tests/unit/match-detail.test.ts @@ -15,8 +15,8 @@ beforeEach(async () => { received = 0; fixture = await newFixture([]); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers (name, komodo_name, game, map) VALUES (?, ?, 't5', 'zombie_theater')") - .run(server, server); + db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't5', 'zombie_theater')") + .run(server); }); afterEach(async () => fixture.cleanup()); @@ -89,7 +89,7 @@ describe('completed match details', () => { db().prepare('DELETE FROM servers WHERE name = ?').run(server); const deleted = getMatchDetail(1)!; expect(deleted).toEqual({ ...before, server_deleted_at: expect.any(String) }); - db().prepare("INSERT INTO servers (name, komodo_name, game, map) VALUES (?, 'replacement', 't6', 'zm_transit')").run(server); + db().prepare("INSERT INTO servers (name, game, map) VALUES (?, 't6', 'zm_transit')").run(server); const { recordLeaderboardTelemetry } = await import('../../src/lib/server/fleet/leaderboard-ingest.js'); recordLeaderboardTelemetry(db(), { event: 'game_end', server, map: 'zm_transit', round: 20, players: [player('999', { kills: 999 })], receivedAt: timestamp(4) }); diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts index b8754e8..96f510b 100644 --- a/tests/unit/metrics.test.ts +++ b/tests/unit/metrics.test.ts @@ -64,8 +64,8 @@ describe('historical metrics', () => { const at = Date.UTC(2026, 0, 2); expect(readMetrics(1, at)).toMatchObject({ firstRetainedSampleAt: null, latestSampleAt: null, registeredServerCount: 0, servers: [], maps: [] }); - db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('current', 'current', null); - db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('removed', 'removed', new Date(at).toISOString()); + db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('current', null); + db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('removed', new Date(at).toISOString()); const sample = { server: 'removed', game: 't6', map: 'Town', players: 1, capacity: null }; recordMetrics([sample], at); const emptyRange = readMetrics(1, at + 2 * 86400_000); @@ -341,8 +341,8 @@ describe('historical metrics', () => { recordMetrics([sample, { ...sample, server: 'removed' }, { ...sample, server: 'tombstone' }], at); // A newly registered name can link to the current server without asserting // that its earlier historical observations belong to this registration. - db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run('reused', 'reused', new Date(at + 15_000).toISOString()); - db().prepare('INSERT INTO servers (name, komodo_name, destroyed_at) VALUES (?, ?, ?)').run('tombstone', 'tombstone', new Date(at).toISOString()); + db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run('reused', new Date(at + 15_000).toISOString()); + db().prepare('INSERT INTO servers (name, destroyed_at) VALUES (?, ?)').run('tombstone', new Date(at).toISOString()); const result = readMetrics(1, at + 30_000); expect(result.servers.find(server => server.server === 'reused')!.currentRegistered).toBe(true); expect(result.servers.filter(server => server.server !== 'reused').every(server => !server.currentRegistered)).toBe(true); diff --git a/tests/unit/native-creation-intents.test.ts b/tests/unit/native-creation-intents.test.ts index 46347a0..5da7f77 100644 --- a/tests/unit/native-creation-intents.test.ts +++ b/tests/unit/native-creation-intents.test.ts @@ -10,7 +10,7 @@ const server = 't5_zombie_theater_1'; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + vi.stubEnv('RCON_PASSWORD', ''); }); afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); @@ -30,12 +30,12 @@ async function intent() { } describe('native creation intents before placement', () => { - it('retains its identity and native history across restart without falling into legacy provisioning', async () => { + it('retains its identity and native history across restart without changing ownership', async () => { const id = await intent(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js'); - const { nextLobbyNumber, processCreations } = await import('../../src/lib/server/provisioning/instances.js'); + const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); expect(nativeDeployment(id)).toBe(true); expect(unplacedNativeCreation(server)).toBe(id); @@ -43,7 +43,8 @@ describe('native creation intents before placement', () => { expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 }); closeDb(); interruptDeployments(); - await processCreations(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); expect(getDeployment(id)).toMatchObject({ backend: 'native', canCancelCreation: true, agentOperation: null, deployment: { state: 'queued', server_exists: 0 } }); expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null }); @@ -78,7 +79,7 @@ describe('native creation intents before placement', () => { it('will not cancel an unplaced historical intent against a replacement server', async () => { const id = await intent(); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers(name,komodo_name) VALUES(?,'replacement')").run(server); + db().prepare("INSERT INTO servers(name) VALUES(?)").run(server); const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(canCancelCreation(id)).toBe(false); expect(cancelCreation(id)).toBeNull(); diff --git a/tests/unit/native-drift-manifest.test.ts b/tests/unit/native-drift-manifest.test.ts index b50ffe6..50e27c2 100644 --- a/tests/unit/native-drift-manifest.test.ts +++ b/tests/unit/native-drift-manifest.test.ts @@ -22,8 +22,8 @@ async function setup(capable = true) { }); const { db } = await import('../../src/lib/server/db/index.js'); const database = db(); - database.prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,created_by) - VALUES(?,'','t5','zombie_theater','zombie','theater','ui')`).run(server); + database.prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,created_by) + VALUES(?,'t5','zombie_theater','zombie','theater','ui')`).run(server); database.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'library','image',28960,?)`).run(randomUUID(), server, host.id, new Date().toISOString()); diff --git a/tests/unit/native-launch-preparation.test.ts b/tests/unit/native-launch-preparation.test.ts index 88e75c4..f9f7795 100644 --- a/tests/unit/native-launch-preparation.test.ts +++ b/tests/unit/native-launch-preparation.test.ts @@ -19,7 +19,7 @@ beforeEach(async () => { fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id; const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3, diff --git a/tests/unit/native-observations.test.ts b/tests/unit/native-observations.test.ts index ff986a3..2281133 100644 --- a/tests/unit/native-observations.test.ts +++ b/tests/unit/native-observations.test.ts @@ -10,7 +10,7 @@ async function setup() { const host = hosts.createHost({ name: 'Observation host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { protocolVersion: 1 }); const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers(name,komodo_name) VALUES('test','')").run(); + db().prepare("INSERT INTO servers(name) VALUES('test')").run(); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES('instance','test',?,'library','image',28960,?)`).run(host.id, new Date().toISOString()); const native = await import('../../src/lib/server/backends/native.js'); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index bf7c095..15ea82f 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -19,7 +19,7 @@ beforeEach(async () => { fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id; enrollHost(createEnrollment(hostId).token, inventory()); @@ -250,7 +250,6 @@ describe('native instance provisioning', () => { expect(result.progress?.checks[0].source).toBe('agent'); const steps = provisioningGroups(result).flatMap(group => group.steps); expect(steps.every(step => step.state === 'done')).toBe(true); - expect(steps.some(step => step.label.includes('Komodo'))).toBe(false); expect(result.agentOperation?.state).toBe('completed'); }); @@ -410,7 +409,7 @@ describe('native instance provisioning', () => { expect(nativeInstance(job.name)?.library_id).toBe(acceptedId); }); - it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => { + it.each(['t5', 't6'] as const)('queues and verifies a %s deployment through its host agent', async game => { const job = await queue(game); await advance(); const operation = await operationFor(job.deploymentId); expect(job.port).toBeNull(); @@ -498,8 +497,8 @@ describe('native instance provisioning', () => { const { db } = await import('../../src/lib/server/db/index.js'); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' }); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); // Version evidence must follow the submitted command, even on a fast test host. db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); @@ -526,8 +525,8 @@ describe('native instance provisioning', () => { const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' }); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' }); const submittedAt = new Date(Date.now() - 1000).toISOString(); db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId); db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId); diff --git a/tests/unit/native-registry-migration.test.ts b/tests/unit/native-registry-migration.test.ts new file mode 100644 index 0000000..a5976fa --- /dev/null +++ b/tests/unit/native-registry-migration.test.ts @@ -0,0 +1,51 @@ +import Database from 'better-sqlite3'; +import { readFile } from 'node:fs/promises'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { await fixture.cleanup(); }); + +it('retires only obsolete backend columns while preserving instances, history, keys and foreign keys', async () => { + const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8'); + const previous = new Database(fixture.dbPath); + // Model the former required resource identity and an unrelated extension. + previous.exec(schema); + previous.exec(`ALTER TABLE servers ADD COLUMN komodo_type TEXT NOT NULL DEFAULT 'deployment'; + ALTER TABLE servers ADD COLUMN komodo_name TEXT NOT NULL; + ALTER TABLE servers ADD COLUMN komodo_server TEXT; + ALTER TABLE servers ADD COLUMN operator_note TEXT;`); + previous.prepare(`INSERT INTO servers(name,komodo_name,komodo_server,game,game_port,operator_note) + VALUES('native-server','','old-host','t5',28960,'keep this')`).run(); + previous.prepare("INSERT INTO server_keys(server_id,pluto_key) VALUES('native-server','saved-key')").run(); + previous.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('creation','native-server','create','done','Ready','2026-01-01','fixture')`).run(); + previous.prepare(`INSERT INTO server_deployment_events(deployment_id,created_at,message) + VALUES('creation','2026-01-01','Original receipt')`).run(); + const { migrateNativeInstances } = await import('../../src/lib/server/backends/schema.js'); + migrateNativeInstances(previous); + previous.prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('instance','native-server','host','library','image',28960,'2026-01-01')`).run(); + previous.close(); + + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + for (let pass = 0; pass < 2; pass++) { + const handle = db(); + const columns = (handle.prepare('PRAGMA table_info(servers)').all() as { name: string }[]).map(column => column.name); + expect(columns).not.toEqual(expect.arrayContaining(['komodo_type'])); + expect(columns).not.toContain('komodo_name'); + expect(columns).not.toContain('komodo_server'); + expect(columns).toContain('operator_note'); + expect(handle.prepare('SELECT name,game_port,operator_note FROM servers WHERE name=?').get('native-server')) + .toEqual({ name: 'native-server', game_port: 28960, operator_note: 'keep this' }); + expect(handle.prepare('SELECT id,generation FROM native_instances').all()).toEqual([{ id: 'instance', generation: 1 }]); + expect(handle.prepare('SELECT pluto_key FROM server_keys').get()).toEqual({ pluto_key: 'saved-key' }); + expect(handle.prepare('SELECT message FROM server_deployment_events').get()).toEqual({ message: 'Original receipt' }); + expect(handle.pragma('foreign_key_check')).toEqual([]); + expect(handle.pragma('foreign_keys', { simple: true })).toBe(1); + closeDb(); + } + // Native inserts must work without supplying the retired required field. + expect(() => db().prepare("INSERT INTO servers(name) VALUES('new-native-server')").run()).not.toThrow(); +}); diff --git a/tests/unit/native-safety.test.ts b/tests/unit/native-safety.test.ts index b90207a..123ab6e 100644 --- a/tests/unit/native-safety.test.ts +++ b/tests/unit/native-safety.test.ts @@ -13,14 +13,14 @@ const server = 't5_zombie_theater_1'; const healthy = { exists: true, healthy: true, state: 'running' }; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); - vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Safety test host' }).id; enrollHost(createEnrollment(hostId).token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: ['plutainer:test'], libraries: [{ id: 'published-game-version', state: 'ready' }] }); const { db } = await import('../../src/lib/server/db/index.js'); instanceId = randomUUID(); - db().prepare("INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'','t5','zombie_theater','zombie','theater',28960,'running')").run(server); + db().prepare("INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state) VALUES(?,'t5','zombie_theater','zombie','theater',28960,'running')").run(server); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,'published-game-version','plutainer:test',28960,?)`).run(instanceId, server, hostId, new Date().toISOString()); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { @@ -31,8 +31,8 @@ beforeEach(async () => { afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); async function advance() { - const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); - await processNativeProvisioning(() => null); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); } async function operationId(id: string) { const { db } = await import('../../src/lib/server/db/index.js'); @@ -119,3 +119,56 @@ describe('native disruption safety at dispatch', () => { expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.restart'").get()).toEqual({ count: 0 }); }); }); + +describe('native scheduler ownership', () => { + it('rejects lifecycle commands for a registry row without a host instance', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + db().prepare("INSERT INTO servers(name) VALUES('unplaced-registry')").run(); + expect(requestLifecycle('unplaced-registry', 'restart')).toMatchObject({ ok: false, status: 503 }); + expect(destroyServer('unplaced-registry')).toMatchObject({ ok: false, kind: 'conflict' }); + expect(db().prepare("SELECT id FROM server_deployments WHERE server='unplaced-registry'").all()).toEqual([]); + }); + + it('shares restart capacity until an agent receipt and fresh health release the first slot', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 }); + db().prepare("INSERT INTO servers(name,game,map,game_port) VALUES('second','t5','zombie_theater',28961)").run(); + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + SELECT 'second-instance','second',host_id,library_id,image,28961,created_at FROM native_instances WHERE id=?`).run(instanceId); + const first = await waitingRestart(); + const second = requestLifecycle('second', 'restart'); + if (!second.ok) throw new Error(second.error); + await advance(); + const command = await operationId(first); + expect(command).toEqual(expect.any(String)); + expect(await operationId(second.deploymentId)).toBeNull(); + await advance(); + expect(await operationId(first)).toBe(command); + expect(await operationId(second.deploymentId)).toBeNull(); + reportHostOperation(hostId, command!, { state: 'completed', result: healthy }); + await advance(); + await advance(); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(first)).toEqual({ state: 'done' }); + expect(await operationId(second.deploymentId)).toEqual(expect.any(String)); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toHaveLength(2); + }); + + it('holds a paused campaign and cancels withdrawn work before delivering a restart', async () => { + const id = await stageCampaign(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE deployment_servers SET restart_state='held' WHERE deployment_id='campaign-race'").run(); + await advance(); + expect(await operationId(id)).toBeNull(); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'waiting' }); + db().prepare("UPDATE deployment_servers SET restart_state='done' WHERE deployment_id='campaign-race'").run(); + await advance(); + expect(await operationId(id)).toBeNull(); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' }); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]); + }); +}); diff --git a/tests/unit/operation-history.test.ts b/tests/unit/operation-history.test.ts index a15e488..4aef20e 100644 --- a/tests/unit/operation-history.test.ts +++ b/tests/unit/operation-history.test.ts @@ -10,7 +10,7 @@ describe('persistent activity', () => { it('keeps beta failures and retries across reopening without duplicating polls', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run(); handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run(); handle.prepare("INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'running')").run(); @@ -29,7 +29,7 @@ describe('persistent activity', () => { it('records campaign files, restart and health milestones, and cascades cleanup', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run(); + handle.prepare("INSERT INTO servers (name) VALUES ('server')").run(); handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', '2026-09-17T00:00:00Z')").run(); handle.prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path) VALUES ('campaign', 'server', 'updated', 'test.gsc')").run(); handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', 'server', 1, 'pending')").run(); diff --git a/tests/unit/plugin-versions.test.ts b/tests/unit/plugin-versions.test.ts index 3940824..5f90308 100644 --- a/tests/unit/plugin-versions.test.ts +++ b/tests/unit/plugin-versions.test.ts @@ -5,7 +5,7 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; }); afterEach(async () => { await fixture.cleanup(); }); diff --git a/tests/unit/plutonium.test.ts b/tests/unit/plutonium.test.ts index 0b17e78..1180398 100644 --- a/tests/unit/plutonium.test.ts +++ b/tests/unit/plutonium.test.ts @@ -3,7 +3,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); let fixture: Fixture; @@ -13,15 +12,15 @@ beforeEach(async () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-21T05:59:00Z')); fixture = await newFixture(servers); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); + vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - const k = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true }); - vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); + const k = await import('../../src/lib/server/backends/native.js'); + + vi.mocked(k.nativeHealth).mockResolvedValue({ ok: true, state: 'running' }); const r = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0); vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); @@ -38,18 +37,18 @@ describe('Plutonium versions', () => { const m = await api(); await m.checkPlutonium(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString()); closeDb(); - vi.mocked(komodoInspectHealthy).mockClear(); + vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); vi.setSystemTime(Date.now() + 1_000); await m.checkPlutonium(); - expect(komodoInspectHealthy).toHaveBeenCalledTimes(1); - expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]); + expect(nativeHealth).toHaveBeenCalledTimes(1); + expect(nativeHealth).toHaveBeenCalledWith(servers[1]); expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1); expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]); expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); @@ -63,17 +62,17 @@ describe('Plutonium versions', () => { const m = await api(); await m.checkPlutonium(); const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString()); db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') .run(servers[0], new Date().toISOString(), 'failed-restart'); - vi.mocked(komodoInspectHealthy).mockClear(); + vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); await m.checkPlutonium(); - expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]); + expect(nativeHealth).not.toHaveBeenCalledWith(servers[0]); expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]); expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved); db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]); @@ -86,13 +85,13 @@ describe('Plutonium versions', () => { await m.checkPlutonium(); const { db } = await import('../../src/lib/server/db/index.js'); const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]); const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString()); vi.mocked(rconPlutoniumVersion).mockClear(); - if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => { + if (phase === 'health') vi.mocked(nativeHealth).mockImplementationOnce(async () => { queue(); return { ok: true, state: 'running' }; }); else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { @@ -105,7 +104,7 @@ describe('Plutonium versions', () => { }); it('coalesces published-release checks without probing the fleet', async () => { const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); const first = checkRelease(); const second = checkRelease(); @@ -113,7 +112,7 @@ describe('Plutonium versions', () => { await Promise.all([first, second]); expect(fetch).toHaveBeenCalledTimes(1); expect(release()).toMatchObject({ revision: 5354, error: null }); - expect(komodoInspectHealthy).not.toHaveBeenCalled(); + expect(nativeHealth).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); await checkRelease(); expect(fetch).toHaveBeenCalledTimes(2); @@ -266,7 +265,7 @@ describe('Plutonium versions', () => { // Both registrations have no lifecycle submission, so only their // registry identity distinguishes the old and replacement instances. db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); - db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now); + db().prepare('INSERT INTO servers (name, created_at) VALUES (?, ?)').run(servers[0], now); } vi.setSystemTime(Date.now() + 1); expect(await probeVersion(servers[0])).toBe(5354); @@ -314,103 +313,9 @@ describe('maintenance timezones', () => { expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00'); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z'); }); - it('persists the zone and advances in local calendar days across DST', async () => { - vi.setSystemTime(new Date('2026-03-07T10:59:00Z')); - const m = await api(); await m.checkPlutonium(); - m.saveMaintenanceSchedule(true, '06:00', 'America/New_York'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); - expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z'); - vi.setSystemTime(new Date('2026-03-07T11:00:00Z')); - await maintenanceTick(); - expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' }); - }); }); describe('persistent maintenance', () => { - it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - await m.maintenanceTick(); - expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled(); - const job = m.plutoniumStatus().jobs[0]; - expect(job.provisioning_id).toBeTruthy(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - // The source has not mirrored submission yet; cancellation must still - // consult the authoritative operation, not its stale queued state. - expect(m.plutoniumStatus().jobs[0].state).toBe('queued'); - expect(m.cancelMaintenance(job.id)).toBe(false); - expect(komodoRestart).toHaveBeenCalledTimes(1); - vi.setSystemTime(Date.now() + 15_000); await processProvisioning(); - await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('done'); - }); - it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => { - const m = await api(); await m.checkPlutonium(); - expect(m.queueMaintenance([servers[0]])).toBe(1); - expect(m.queueMaintenance([servers[0]])).toBe(0); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null); - await maintenanceTick(); await maintenanceTick(); - expect(komodoRestart).not.toHaveBeenCalled(); - await Promise.all([maintenanceTick(), maintenanceTick()]); - expect(komodoRestart).toHaveBeenCalledTimes(1); - expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); - await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); - vi.setSystemTime(Date.now() + 15_000); - await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' }); - }); - it('shares rolling capacity across servers and survives closing the database', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers); - await maintenanceTick(); - const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); - expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']); - }); - it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - await maintenanceTick(); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 }); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - it('does not retry an ambiguous restart timeout', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' }); - await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('failed'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - it('honors cancellation arriving during the player check', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { - m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0; - }); - await maintenanceTick(); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled'); - }); - it('holds maintenance while lifecycle work is queued', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) - VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString()); - await maintenanceTick(); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation'); - }); it('waits to attach legacy maintenance while another operation owns the server', async () => { const m = await api(); await m.checkPlutonium(); const { db } = await import('../../src/lib/server/db/index.js'); @@ -425,29 +330,6 @@ describe('persistent maintenance', () => { expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy(); expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 }); }); - it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => { - const m = await api(); await m.checkPlutonium(); - const { db } = await import('../../src/lib/server/db/index.js'); - const submittedAt = new Date().toISOString(); - db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state) - VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt); - if (linked) { - db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); - } - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') - .run(servers[0], submittedAt, linked ? 'legacy-restart' : null); - await m.maintenanceTick(); - const job = m.plutoniumStatus().jobs[0]; - expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' }); - expect(job.provisioning_id).toBeTruthy(); - if (linked) expect(job.provisioning_id).toBe('legacy-restart'); - expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id }); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('done'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).not.toHaveBeenCalled(); - }); it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); @@ -455,12 +337,12 @@ describe('persistent maintenance', () => { db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id); db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString()); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); + vi.mocked(nativeHealth).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') }); - expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect(nativeHealth).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('done'); @@ -493,18 +375,6 @@ describe('persistent maintenance', () => { await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' }); }); - it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => { - const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run(); - await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).not.toHaveBeenCalled(); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0].state).toBe('done'); - expect(komodoRestart).not.toHaveBeenCalled(); - }); it('validates API requests without changing the schedule or queue', async () => { const { POST } = await import('../../src/routes/api/plutonium/+server.js'); const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', { @@ -521,19 +391,6 @@ describe('persistent maintenance', () => { expect(m.plutoniumStatus().schedule.enabled).toBe(true); expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' }); }); - it('runs a daily UTC schedule once and coalesces missed days', async () => { - expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z'); - expect(() => nextMaintenanceAt('24:00')).toThrow(); - const m = await api(); await m.checkPlutonium(); - expect(m.plutoniumStatus().schedule.enabled).toBe(false); - m.saveMaintenanceSchedule(true, '06:00', 'UTC'); - vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); - await maintenanceTick(); await maintenanceTick(); - expect(m.plutoniumStatus().jobs).toHaveLength(2); - expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z'); - m.saveMaintenanceSchedule(false, '06:00', 'UTC'); - expect(m.plutoniumStatus().schedule.nextAt).toBeNull(); - }); }); describe('Maintenance policies and runtime updates', () => { @@ -614,64 +471,6 @@ describe('Maintenance policies and runtime updates', () => { expect(m.queueUpdates([servers[0]])).toBe(1); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' }); }); - it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => { - const m = await outdated(); - const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false); - await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled(); - p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true); - await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled(); - p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1); - p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000); - await processProvisioning(); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' }); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => { - const m = await outdated(); - const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]); - await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - const failed = m.plutoniumStatus().jobs[0]; - expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' }); - closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs).toHaveLength(1); - expect(m.retryMaintenance(failed.id)).toBe(1); - expect(m.plutoniumStatus().jobs).toHaveLength(2); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); - await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' }); - expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined(); - }); - it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => { - const m = await outdated(); m.queueUpdates([servers[0]]); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE servers SET desired_state = 'stopped'").run(); - await maintenanceTick(); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled'); - expect(m.queueUpdates()).toBe(0); - expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true); - }); - it('separates scheduled restart success from revision verification and coalesces paused runs', async () => { - const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); - const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true); - vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick(); - expect(m.plutoniumStatus().jobs).toHaveLength(0); - p.pauseMaintenance(false); await maintenanceTick(); - expect(m.plutoniumStatus().jobs).toHaveLength(2); - expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' }); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done'); - expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z'); - }); it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); @@ -679,25 +478,12 @@ describe('Maintenance policies and runtime updates', () => { expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown'); vi.setSystemTime(Date.now() + 3_600_000); expect(() => m.queueUpdates()).toThrow('published revision'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockClear(); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); + vi.mocked(nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); await m.checkPlutonium(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true); }); - it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => { - const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York'); - m.queueMaintenance([servers[0]]); await maintenanceTick(); - const before = m.plutoniumStatus(); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb(); - const after = m.plutoniumStatus(); - expect(after.schedule).toEqual(before.schedule); - expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' }); - vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); }); diff --git a/tests/unit/provisioning-cancellation.test.ts b/tests/unit/provisioning-cancellation.test.ts index c738f83..38aed39 100644 --- a/tests/unit/provisioning-cancellation.test.ts +++ b/tests/unit/provisioning-cancellation.test.ts @@ -32,7 +32,7 @@ describe('provisioning cancellation', () => { const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - db().prepare("INSERT INTO servers(name,komodo_name) VALUES('one','one'),('two','two')").run(); + db().prepare("INSERT INTO servers(name) VALUES('one'),('two')").run(); const id = beginDeployment('one','restart'); const other = beginDeployment('two','restart'); db().prepare("INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES('one','2026-10-05',?),('two','2026-10-05',?)").run(id, other); diff --git a/tests/unit/provisioning-detail.test.ts b/tests/unit/provisioning-detail.test.ts index 0077a87..c9dd8d7 100644 --- a/tests/unit/provisioning-detail.test.ts +++ b/tests/unit/provisioning-detail.test.ts @@ -22,9 +22,9 @@ describe('provisioning detail projection', () => { it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => { const history = await import('../../src/lib/server/provisioning/history.js'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); + const backend = await import('../../src/lib/server/backends/native.js'); const rcon = await import('../../src/lib/server/integrations/rcon.js'); - const inspect = vi.spyOn(komodo, 'komodoInspectHealthy'); + const inspect = vi.spyOn(backend, 'nativeHealth'); const status = vi.spyOn(rcon, 'rconStatusResult'); const id = history.beginDeployment('t5_zombie_theater_1', 'create'); for (let index = 0; index < 40; index++) history.recordDeploymentEvent(id, `Recorded activity ${index}`); @@ -73,7 +73,7 @@ describe('provisioning detail projection', () => { it('correlates reservations by operation ID without assigning a later same-name slot to historical work', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { beginDeployment, getDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); - db().prepare("INSERT INTO servers (name,komodo_name) VALUES ('server','server')").run(); + db().prepare("INSERT INTO servers (name) VALUES ('server')").run(); const old = beginDeployment('server', 'restart'); finishDeployment(old, { ok: false, message: 'Timeout' }); const current = beginDeployment('server', 'restart'); @@ -104,7 +104,7 @@ describe('provisioning detail projection', () => { it('preserves a failure outcome and evidence before an observation-only same-ID resume', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const history = await import('../../src/lib/server/provisioning/history.js'); - db().prepare("INSERT INTO servers (name,komodo_name,game_port) VALUES ('server','server',28960)").run(); + db().prepare("INSERT INTO servers (name,game_port) VALUES ('server',28960)").run(); const id = history.beginDeployment('server', 'create'); db().prepare("INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')").run(id); history.deploymentStage(id, 'Waiting for game evidence', { phase: 'verify-game', checks: [ diff --git a/tests/unit/provisioning-evidence.test.ts b/tests/unit/provisioning-evidence.test.ts deleted file mode 100644 index 6592c6a..0000000 --- a/tests/unit/provisioning-evidence.test.ts +++ /dev/null @@ -1,405 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { join } from 'node:path'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { gameEvent } from '../helpers/telemetry.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(), - komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(), - komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn() -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); - -let fixture: Fixture; -const server = 't5_zombie_theater_1'; -const status = (map = 'zombie_theater') => ({ ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '', - status: { players: 0, map, gametype: 'zom', roster: [] } }); - -beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); - vi.useFakeTimers({ toFake: ['Date'] }); - vi.setSystemTime(new Date('2026-09-28T12:00:00Z')); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354 }))); - fixture = await newFixture([server]); - for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test', - KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1', - SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) { - vi.mocked(komodo[name]).mockResolvedValue({ ok: true }); - } - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name })); - vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true }); - const rcon = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0); - vi.mocked(rcon.rconStatusResult).mockResolvedValue(status()); - vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); -}); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); - -async function register() { - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir) - VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server)); - const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - saveServerKey(server, 'saved-key'); -} -async function detail(id: string) { - const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); - return getDeployment(id)!; -} -async function telemetry(players: number, map = 'zombie_theater') { - const { db } = await import('../../src/lib/server/db/index.js'); - const receivedAt = new Date().toISOString(); - db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?) - ON CONFLICT(server) DO UPDATE SET players=excluded.players,received_at=excluded.received_at,payload=excluded.payload`) - .run(server, players, receivedAt, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt })); - return receivedAt; -} - -async function launch(game: 't5' | 't6' = 't5') { - const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconStatusResult).mockResolvedValue(status(game === 't5' ? 'zombie_theater' : 'zm_transit')); - const job = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', plutoKey: 'saved-key' }); - if (!job.ok) throw new Error(job.message); - await processCreations(); - vi.setSystemTime(Date.now() + 15_000); - return { ...job, processCreations }; -} - -describe('Plutonium provisioning verification', () => { - it.each(['t5', 't6'] as const)('verifies production and persists the running %s revision before completing', async game => { - const job = await launch(game); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); - const { PLUTONIUM_MANIFEST_URL } = await import('../../src/lib/plutonium.js'); - expect(rconPlutoniumVersion).not.toHaveBeenCalled(); - await job.processCreations(); - const completed = await detail(job.deploymentId); - expect(completed.deployment.state).toBe('done'); - expect(completed.progress).toMatchObject({ phase: 'verify-plutonium', blocker: null }); - expect(completed.progress?.checks).toEqual(expect.arrayContaining([ - expect.objectContaining({ key: 'plutonium-release', source: 'manifest', result: 'pass', actual: 5354 }), - expect.objectContaining({ key: 'plutonium-version', source: 'rcon', result: 'pass', expected: 5354, actual: 5354 }) - ])); - expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5354, update_status: 'current' }); - expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.objectContaining({ redirect: 'error' })); - expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1); - expect(rconPlutoniumVersion).toHaveBeenCalledWith(job.name); - const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js'); - expect(isProvisioningDetail(completed, job.deploymentId)).toBe(true); - expect(provisioningGroups(completed).find(group => group.key === 'verify')?.state).toBe('done'); - }); - - it('retries unknown runtime evidence while retaining startup capacity and never replays creation', async () => { - const job = await launch(); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); - const next = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second-key' }); - if (!next.ok) throw new Error(next.message); - // A pre-start observation is not proof of the newly started process. - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('INSERT INTO plutonium_versions (server, revision, checked_at) VALUES (?, 5354, ?)') - .run(job.name, '2026-09-28T11:59:00.000Z'); - vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version reply timed out.' }); - await job.processCreations(); - expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health'); - expect((await detail(job.deploymentId)).progress?.blocker).toMatchObject({ code: 'plutonium-version', automatic: true }); - expect((await detail(next.deploymentId)).deployment.state).toBe('queued'); - expect(db().prepare('SELECT revision FROM plutonium_versions WHERE server = ?').get(job.name)).toEqual({ revision: null }); - await job.processCreations(); - expect((await detail(job.deploymentId)).deployment.state).toBe('done'); - expect((await detail(next.deploymentId)).deployment.state).toBe('awaiting-health'); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); - expect(komodoDeploy).toHaveBeenCalledTimes(2); - }); - - it('requires a successful manifest check even with a previously cached revision, then resumes without recreating', async () => { - const job = await launch(); - const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js'); - writeSetting('plutonium_release', { revision: 5300, checkedAt: new Date().toISOString(), error: null }); - vi.mocked(fetch).mockRejectedValueOnce(new Error('Manifest offline')); - await job.processCreations(); - expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health'); - expect((await detail(job.deploymentId)).progress?.checks).toContainEqual(expect.objectContaining({ - key: 'plutonium-release', result: 'unknown', actual: null, message: 'Manifest offline' - })); - await job.processCreations(); - expect((await detail(job.deploymentId)).deployment.state).toBe('done'); - const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - }); - - it('reports a confirmed outdated runtime without automatically restarting or recreating it', async () => { - const job = await launch(); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); - await job.processCreations(); - const failed = await detail(job.deploymentId); - expect(failed.deployment).toMatchObject({ state: 'failed', error: expect.stringContaining('running r5300, production is r5354') }); - expect(failed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'plutonium-version', result: 'fail', expected: 5354, actual: 5300 })); - const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); - expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5300, update_status: 'outdated' }); - await job.processCreations(); - const { komodoCreateDeployment, komodoDeploy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('accepts a runtime ahead of the published release without requesting a downgrade', async () => { - const job = await launch(); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5355, error: null }); - await job.processCreations(); - expect((await detail(job.deploymentId)).deployment.state).toBe('done'); - const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); - expect(runningVersions().find(version => version.server === job.name)?.update_status).toBe('ahead'); - }); - - it('discards a late version response when the instance was removed during the probe', async () => { - const job = await launch(); - const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { - db().prepare('DELETE FROM servers WHERE name = ?').run(job.name); - return { revision: 5354, error: null }; - }); - await job.processCreations(); - const removed = await detail(job.deploymentId); - expect(removed.deployment.server_deleted_at).toBeTruthy(); - expect(removed.deployment.state).not.toBe('done'); - expect(removed.progress?.checkpoints.some(check => check.key === 'verify-plutonium')).toBe(false); - expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(job.name)).toBeUndefined(); - }); -}); - -describe('provisioning decision evidence', () => { - it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => { - const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js'); - const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' }); - if (!job.ok) throw new Error(job.message); - expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'startup-queue', recipeStatus: 'queued', observedAt: null }); - - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { script } = createScript({ name: 'New promotion', targetRelPath: 'promoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('promoted while queued') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id); - await processCreations(); - expect((await detail(job.deploymentId)).progress?.recipeStatus).toBe('attempted'); - expect((await detail(job.deploymentId)).recipe?.files.some(file => file.path === 'promoted.gsc')).toBe(true); - - vi.mocked(rconStatusResult).mockResolvedValueOnce(status('zombie_moon')); - vi.setSystemTime(Date.now() + 1000); - await processCreations(); - expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'verify-map', blocker: { code: 'map-mismatch' }, - checks: [{ key: 'container', result: 'pass' }, { key: 'game', result: 'pass' }, { key: 'map', result: 'waiting', expected: 'zombie_theater', actual: 'zombie_moon' }] }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'container recovering' }); - await processCreations(); - expect((await detail(job.deploymentId)).progress?.checks).toEqual([expect.objectContaining({ key: 'container', result: 'waiting' })]); - - vi.setSystemTime(Date.now() + 1000); - const receivedAt = await telemetry(0); - await processCreations(); - const completed = await detail(job.deploymentId); - expect(completed.deployment.state).toBe('done'); - expect(completed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'game', source: 'telemetry', result: 'pass', receivedAt })); - expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'verify-map' })); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeploy).toHaveBeenCalledTimes(1); - }); - - it('keeps an uncertain failed request immutable while observing and releasing its exact recovery slot', async () => { - await register(); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'response lost' }); - const job = requestLifecycle(server, 'restart'); - if (!job.ok) throw new Error(job.error); - await processProvisioning(); - const failed = await detail(job.deploymentId); - expect(failed.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'held', operationId: job.deploymentId } }); - const failureEvents = failed.events.filter(event => event.kind === 'outcome'); - vi.setSystemTime(Date.now() + 15_000); - await telemetry(0); - await processProvisioning(); - const recovered = await detail(job.deploymentId); - expect(recovered.deployment).toMatchObject({ state: 'failed', finished_at: failed.deployment.finished_at }); - expect(recovered.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'released', operationId: job.deploymentId } }); - expect(recovered.events.filter(event => event.kind === 'outcome')).toEqual(failureEvents); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('records occupied and unknown evidence without treating either as zero or resetting eligibility timing', async () => { - await register(); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const job = requestLifecycle(server, 'restart'); - if (!job.ok) throw new Error(job.error); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null); - await processProvisioning(); - const unknown = await detail(job.deploymentId); - expect(unknown.progress?.checks).toEqual([expect.objectContaining({ key: 'occupancy', actual: null, source: null, result: 'unknown' })]); - vi.setSystemTime(Date.now() + 1000); - await telemetry(3); - await processProvisioning(); - expect((await detail(job.deploymentId)).progress).toMatchObject({ phaseStartedAt: unknown.progress?.phaseStartedAt, - blocker: { code: 'players-present' }, checks: [{ key: 'occupancy', actual: 3, source: 'telemetry' }] }); - vi.setSystemTime(Date.now() + 1000); - await telemetry(2); - await processProvisioning(); - expect((await detail(job.deploymentId)).progress?.phaseStartedAt).toBe(unknown.progress?.phaseStartedAt); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('preserves failed cleanup evidence through same-ID resume without another destructive command', async () => { - await register(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js'); - const { resumeCleanup } = await import('../../src/lib/server/provisioning/history.js'); - const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'cleanup interrupted'); END"); - const job = destroyServer(server); - if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal'); - await processDestructions(); - const failed = await detail(job.deploymentId); - expect(failed.deployment.state).toBe('failed'); - expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' })); - expect(failed.progress?.checkpoints.some(point => point.key === 'cleanup')).toBe(false); - const failure = failed.events.find(event => event.kind === 'outcome')!; - db().exec('DROP TRIGGER fail_cleanup'); - vi.setSystemTime(Date.now() + 1000); - expect(resumeCleanup(job.deploymentId)).toBe(true); - await processDestructions(); - const completed = await detail(job.deploymentId); - expect(completed.deployment).toMatchObject({ id: job.deploymentId, state: 'done', server_exists: 0 }); - expect(completed.events).toContainEqual(failure); - expect(completed.events.some(event => event.kind === 'resumed')).toBe(true); - expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'cleanup' })); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1); - }); - - it('links the actual removal dependency while later eligible creation bypasses it', async () => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js'); - const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' }; - const first = queueCreateServer(input), second = queueCreateServer(input); - if (!first.ok || !second.ok) throw new Error('Could not queue creation'); - const removal = beginDeployment('old-instance', 'destroy'); - db().prepare('INSERT INTO server_creation_dependencies VALUES (?, ?)').run(first.deploymentId, removal); - await processCreations(); - expect((await detail(first.deploymentId)).progress?.blocker).toMatchObject({ code: 'removal-dependency', href: `/provisioning/${removal}` }); - expect((await detail(first.deploymentId)).deployment.state).toBe('queued'); - expect((await detail(second.deploymentId)).deployment.state).toBe('awaiting-health'); - }); - - it.each(['create', 'delete'] as const)('records uncertainty when the %s reply throws without inventing a successful checkpoint', async command => { - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - const { queueCreateServer, processCreations, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js'); - let id: string; - if (command === 'create') { - vi.mocked(komodo.komodoCreateDeployment).mockRejectedValueOnce(new Error('reply lost')); - const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' }); - if (!job.ok) throw new Error(job.message); - id = job.deploymentId; - await processCreations(); - expect(komodo.komodoCreateDeployment).toHaveBeenCalledTimes(1); - } else { - await register(); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - vi.mocked(komodo.komodoDeleteDeployment).mockRejectedValueOnce(new Error('reply lost')); - const job = destroyServer(server); - if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal'); - id = job.deploymentId; - await processDestructions(); - expect(komodo.komodoDeleteDeployment).toHaveBeenCalledTimes(1); - } - const failed = await detail(id); - expect(failed.deployment.state).toBe('failed'); - expect(failed.progress?.command).toMatchObject({ state: 'uncertain', message: expect.stringContaining('reply lost') }); - expect(failed.progress?.checkpoints.some(point => point.key === (command === 'create' ? 'create-deployment' : 'delete-deployment'))).toBe(false); - }); - - it.each(['destroy', 'rebuild'] as const)('retains the resumable %s checkpoint when a beta arrives after confirmed remote deletion', async action => { - await register(); - const { db } = await import('../../src/lib/server/db/index.js'); - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js'); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { resumeCleanup, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js'); - const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - vi.mocked(komodoDeleteDeployment).mockImplementationOnce(async () => { - const { script, version } = createScript({ name: 'Late beta', targetRelPath: 'beta.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta') }); - const beta = db().prepare("INSERT INTO betas (script_id,script_version_id,trigger_kind,state) VALUES (?,?,'manual','running')").run(script.id, version.id).lastInsertRowid; - db().prepare("INSERT INTO beta_servers (beta_id,server,had_original,stage) VALUES (?,?,0,'ready')").run(beta, server); - return { ok: true }; - }); - const job = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild'); - if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal'); - await (action === 'destroy' ? processDestructions() : processProvisioning()); - const failed = await detail(job.deploymentId); - expect(failed.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER }); - expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' })); - expect(action === 'destroy' ? failed.canResumeCleanup : failed.canResumeRebuild).toBe(true); - db().prepare("UPDATE betas SET state = 'rolled-back'").run(); - expect(action === 'destroy' ? resumeCleanup(job.deploymentId) : resumeRebuild(job.deploymentId)).toBe(true); - await (action === 'destroy' ? processDestructions() : processProvisioning()); - expect((await detail(job.deploymentId)).deployment.state).toBe(action === 'destroy' ? 'done' : 'queued'); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1); - }); - - it('ignores occupancy evidence that returns after the request was cancelled', async () => { - await register(); - const { requestLifecycle, cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const job = requestLifecycle(server, 'restart'); - if (!job.ok) throw new Error(job.error); - vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { - expect(cancelProvisioning(job.deploymentId)).toBe(true); - return 4; - }); - await processProvisioning(); - const cancelled = await detail(job.deploymentId); - expect(cancelled.deployment.state).toBe('cancelled'); - expect(cancelled.progress).toMatchObject({ checks: [], blocker: null }); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('does not attach replacement health to a deleted creation after a delayed probe', async () => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' }); - if (!job.ok) throw new Error(job.message); - await processCreations(); - vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => { - db().prepare('DELETE FROM servers WHERE name = ?').run(server); - await register(); - return { ok: true, state: 'running' }; - }); - await processCreations(); - const old = await detail(job.deploymentId); - expect(old.deployment.server_deleted_at).toBeTruthy(); - expect(old.progress?.checks).toEqual([]); - expect(old.progress?.checkpoints.some(point => point.key.startsWith('verify-'))).toBe(false); - expect(rconStatusResult).not.toHaveBeenCalled(); - }); -}); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index c118b1f..d2f84b8 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -24,9 +24,9 @@ describe('provisioning reads with retained lifecycle history', () => { const now = Date.parse('2026-10-05T12:00:00Z'); const iso = (offset: number) => new Date(now + offset).toISOString(); handle.transaction(() => { - const server = handle.prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)'); - for (let i = 0; i < 16; i++) server.run(`server-${i}`, `server-${i}`); - server.run('empty', 'empty'); + const server = handle.prepare('INSERT INTO servers(name) VALUES (?)'); + for (let i = 0; i < 16; i++) server.run(`server-${i}`); + server.run('empty'); const insert = handle.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, submitted_at, server_deleted_at, triggered_by) VALUES (?, ?, ?, 'done', 'Complete', ?, ?, ?, ?, 'test')`); @@ -121,7 +121,7 @@ describe('provisioning reads with retained lifecycle history', () => { legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, table => table .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'") .replace(' server_deleted_at TEXT,\n', ''))); - legacy.exec(`INSERT INTO servers(name,komodo_name) VALUES('existing','existing'); + legacy.exec(`INSERT INTO servers(name) VALUES('existing'); INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('old','existing','create','done','Complete','2025-01-01','fixture'); INSERT INTO server_deployment_events(deployment_id,created_at,message) diff --git a/tests/unit/provisioning-presentation.test.ts b/tests/unit/provisioning-presentation.test.ts index 1e32fe0..957b2dc 100644 --- a/tests/unit/provisioning-presentation.test.ts +++ b/tests/unit/provisioning-presentation.test.ts @@ -72,7 +72,7 @@ describe('provisioning outcome and evidence presentation', () => { it('shows passed container and telemetry game checks while waiting on the engine map', () => { const detail = fixture(); detail.progress = progress({ checks: [ - { key: 'container', label: 'Container readiness', result: 'pass', source: 'komodo', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null }, + { key: 'container', label: 'Container readiness', result: 'pass', source: 'agent', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null }, { key: 'game', label: 'Game status', result: 'pass', source: 'telemetry', checkedAt: at, receivedAt: at, expected: null, actual: null, message: 'Fresh telemetry accepted' }, { key: 'map', label: 'Engine map', result: 'waiting', source: 'telemetry', checkedAt: at, receivedAt: at, expected: 'zombie_theater', actual: 'zombie_moon', message: null } ] }); @@ -131,14 +131,13 @@ describe('provisioning outcome and evidence presentation', () => { expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')?.state).toBe('done'); }); - it('keeps removal checkpoints separate from replacement creation', () => { - const detail = fixture(); detail.deployment.action = 'rebuild'; detail.deployment.state = 'waiting'; - detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'delete-deployment', label: 'Remote deployment deleted', at, phase: 'delete-deployment' }] }); + it('keeps confirmed agent removal separate from registry cleanup', () => { + const detail = fixture(); detail.deployment.action = 'destroy'; detail.deployment.state = 'waiting'; + detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'container-action', label: 'Agent confirmed removal', at, phase: 'container-action' }] }); const groups = provisioningGroups(detail); - expect(groups.map(group => group.key)).toEqual(['request', 'remove', 'create', 'verify']); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'delete-deployment')?.state).toBe('done'); + expect(groups.map(group => group.key)).toEqual(['request', 'command', 'cleanup']); + expect(groups.flatMap(group => group.steps).find(step => step.key === 'container-action')?.state).toBe('done'); expect(groups.flatMap(group => group.steps).find(step => step.key === 'cleanup')?.state).toBe('waiting'); - expect(groups.flatMap(group => group.steps).find(step => step.key === 'prepare')?.state).toBe('pending'); }); }); diff --git a/tests/unit/provisioning-progress.test.ts b/tests/unit/provisioning-progress.test.ts index 4ab9953..c69b393 100644 --- a/tests/unit/provisioning-progress.test.ts +++ b/tests/unit/provisioning-progress.test.ts @@ -43,10 +43,10 @@ describe('provisioning evidence persistence', () => { const { handle, recordProgress, read } = await setup(); recordProgress('operation', { phase: 'verify-game', command: { state: 'uncertain', message: 'Reply was lost' }, checkpoint: { key: 'container-action', label: 'Submission attempted', phase: 'container-action' }, - checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'pass' }, + checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'pass' }, { key: 'game', label: 'Game', source: 'telemetry', result: 'pass' }] }); vi.setSystemTime(Date.now() + 15_000); - recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'unknown' }] }); + recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'agent', result: 'unknown' }] }); expect(read()?.checks).toHaveLength(1); expect(read()?.checks[0].result).toBe('unknown'); expect(read()?.checkpoints).toHaveLength(1); @@ -87,7 +87,7 @@ describe('provisioning evidence persistence', () => { const { handle, recordProgress, read } = await setup(); expect(() => handle.transaction(() => { handle.prepare("UPDATE server_deployments SET state = 'running' WHERE id = 'operation'").run(); - recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'delete-deployment', label: 'Remote deletion confirmed' } }); + recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Remote deletion confirmed' } }); throw new Error('Interrupted transaction'); })()).toThrow('Interrupted transaction'); expect(read()).toBeNull(); diff --git a/tests/unit/provisioning.test.ts b/tests/unit/provisioning.test.ts deleted file mode 100644 index acbb1b1..0000000 --- a/tests/unit/provisioning.test.ts +++ /dev/null @@ -1,561 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import Database from 'better-sqlite3'; -import { readFile, writeFile, access } from 'node:fs/promises'; -import { join } from 'node:path'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(), - komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(), - komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), - komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); - -let fixture: Fixture; -const server = 't5_zombie_theater_2'; -const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '', - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }; - -beforeEach(async () => { - vi.resetModules(); vi.resetAllMocks(); - vi.useFakeTimers({ toFake: ['Date'] }); - vi.setSystemTime(new Date('2026-09-27T12:00:00Z')); - fixture = await newFixture([server]); - vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test', - KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1', - SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value); - // Exercise the upgrade path, with history and references present before migration. - const old = new Database(fixture.dbPath); - const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8'); - old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'") - .replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'") - .replace("server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server TEXT NOT NULL REFERENCES servers(name),')); - old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run(); - old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run(); - old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run(); - old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run(); - old.close(); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir) - VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server)); - const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - saveServerKey(server, 'saved-key'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name })); - vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true }); - const { kick } = await import('../../src/lib/server/pipeline/worker.js'); - vi.mocked(kick).mockResolvedValue(undefined); - for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) vi.mocked(komodo[name]).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - const rcon = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0); - vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon); - vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); -}); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); - -async function operation(id: string) { - const { db } = await import('../../src/lib/server/db/index.js'); - return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id); -} - -async function queue(action: 'start' | 'restart' | 'stop' | 'rebuild') { - const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const result = requestLifecycle(server, action); - expect(result.ok).toBe(true); - if (!result.ok) throw new Error(result.error); - return result.deploymentId; -} - -describe('provisioning operations', () => { - it('migrates the action constraint without losing row ordering, activity, or foreign keys', async () => { - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - await queue('start'); - closeDb(); - expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 }); - expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' }); - expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' }); - expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' }); - expect(db().pragma('foreign_key_check')).toEqual([]); - expect(db().pragma('foreign_keys', { simple: true })).toBe(1); - }); - - it('starts without an RCON empty check and verifies recovery after the grace period', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js'); - const id = await queue('start'); - await processLifecycle(); - expect(komodoRuntimeAction).toHaveBeenCalledWith(server, 'start'); - expect(rconGetPlayerCount).not.toHaveBeenCalled(); - expect(await operation(id)).toMatchObject({ action: 'start', state: 'awaiting-health' }); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'awaiting-health' }); - vi.setSystemTime(Date.now() + 15_000); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'done' }); - expect(komodoRuntimeAction).toHaveBeenCalledTimes(1); - }); - - it('deduplicates requests, rejects conflicting actions, and waits for a fresh empty reading', async () => { - const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const id = await queue('restart'); - expect(requestLifecycle(server, 'restart')).toEqual({ ok: true, deploymentId: id }); - expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 }); - expect(destroyServer(server)).toMatchObject({ ok: false, kind: 'conflict' }); - for (const players of [4, null]) { - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'waiting' }); - expect(komodoRestart).not.toHaveBeenCalled(); - } - await processLifecycle(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('does not repeat an uncertain restart and retains its recovery slot', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' }); - const id = await queue('restart'); - await processLifecycle(); await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'failed', error: 'komodo: timeout' }); - expect(komodoRestart).toHaveBeenCalledTimes(1); - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null }); - }); - - it('keeps stop pending on unknown health and completes only on a confirmed stopped state', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { komodoInspectHealthy, komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js'); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - const id = await queue('stop'); - await processLifecycle(); - closeDb(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'network error' }); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'awaiting-health' }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, state: 'exited', reason: 'stopped' }); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'done' }); - expect(komodoRuntimeAction).toHaveBeenCalledTimes(1); - await expect(access(join(fixture.t5, server))).resolves.toBeUndefined(); - }); - - it('rebuilds from nothing in the same slot and port, then verifies the fresh instance', async () => { - const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { processCreations, nextLobbyNumber, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js'); - const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const sentinel = join(fixture.t5, server, 'old-instance.dat'); - await writeFile(sentinel, 'must not survive'); - const id = await queue('rebuild'); - // Promotions after queueing must be included in the fresh instance. - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { script } = createScript({ name: 'Fresh promoted script', targetRelPath: 'fresh.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('fresh content') }); - db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - await processLifecycle(); - expect(komodoDestroyDeployment).toHaveBeenCalledWith(server); - expect(komodoDeleteDeployment).toHaveBeenCalledWith(server); - await expect(access(sentinel)).rejects.toThrow(); - expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined(); - expect(await operation(id)).toMatchObject({ state: 'queued', action: 'rebuild' }); - expect(nextAvailablePort()).toBe(28961); - expect(nextLobbyNumber('t5_zombie_theater', ['t5_zombie_theater_1'])).toBe(3); - expect(requestLifecycle(server, 'rebuild')).toEqual({ ok: true, deploymentId: id }); - await processCreations(); - expect(komodoCreateDeployment).toHaveBeenCalledWith(expect.objectContaining({ name: server, ports: ['28960:28960/udp', '28960:28960/tcp'] })); - await expect(access(sentinel)).rejects.toThrow(); - expect(await operation(id)).toMatchObject({ state: 'awaiting-health' }); - expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.setSystemTime(Date.now() + 1); - await processCreations(); - expect(await operation(id)).toMatchObject({ state: 'done' }); - const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); - expect(runningVersions().find(version => version.server === server)).toMatchObject({ revision: 5354, update_status: 'current' }); - }); - - it('rejects a rebuild without a saved key before touching the existing instance', async () => { - const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - removeServerKey(server); - expect(requestLifecycle(server, 'rebuild')).toMatchObject({ ok: false, error: expect.stringContaining('Save a server key') }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - - it('retains the rebuild cleanup checkpoint and reservation while waiting for capacity', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { beginDeployment, deploymentStage, finishDeployment, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); - const { nextAvailablePort } = await import('../../src/lib/server/fleet/create.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const id = await queue('rebuild'); - // Model a crash after confirmed remote deletion and registry cleanup. - deploymentStage(id, REMOVING_SERVER); - db().prepare('DELETE FROM servers WHERE name = ?').run(server); - finishDeployment(id, { ok: false, message: 'cleanup interrupted' }); - expect(nextAvailablePort()).toBe(28961); - expect(resumeRebuild(id)).toBe(true); - const occupied = beginDeployment('another-server', 'create'); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER }); - closeDb(); - finishDeployment(occupied, { ok: true }); - await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'queued' }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - - it('allows explicit destruction of a stalled startup and records the interrupted verification', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const id = await queue('start'); - await processLifecycle(); - expect(destroyServer(server)).toMatchObject({ ok: true }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); - await processDestructions(); - expect(await operation(id)).toMatchObject({ state: 'cancelled', error: 'Server deleted', server_deleted_at: expect.any(String) }); - }); - - it('validates lifecycle API input and returns the durable operation ID', async () => { - const { POST, GET } = await import('../../src/routes/api/provisioning/+server.js'); - const submit = (body: unknown) => POST({ request: new Request('http://localhost/api/provisioning', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) - }) }); - for (const body of [null, { server, action: 'erase' }, { server: 5, action: 'start' }]) expect((await submit(body)).status).toBe(400); - const result = await submit({ server, action: 'start' }); - expect(result.status).toBe(202); - const { deploymentId } = await result.json(); - expect((await GET({ url: new URL('http://localhost/api/provisioning') }).json()).deployments).toContainEqual(expect.objectContaining({ id: deploymentId, action: 'start', state: 'waiting' })); - }); - - it('does not create a replacement when teardown fails or resume verification of the old instance', async () => { - const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); - const { processCreations } = await import('../../src/lib/server/fleet/create.js'); - const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); - const { komodoDestroyDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' }); - const id = await queue('rebuild'); - await processLifecycle(); await processCreations(); await processLifecycle(); - expect(await operation(id)).toMatchObject({ state: 'failed' }); - expect(getDeployment(id)?.canResumeVerification).toBe(false); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - expect(komodoCreateDeployment).not.toHaveBeenCalled(); - }); -}); - -async function incident(id = 'incident') { - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - // These lifecycle tests exercise an actual container failure. Healthy - // containers with fresh game evidence now cancel obsolete health work. - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' }); - const now = new Date().toISOString(); - db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) - VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now); - return id; -} - -async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') { - const { db } = await import('../../src/lib/server/db/index.js'); - const { gameEvent } = await import('../helpers/telemetry.js'); - db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?) - ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`) - .run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp })); -} - -describe('Watchdog provisioning and telemetry evidence', () => { - it('keeps the original running intent when a queued removal is cancelled', async () => { - const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const removal = destroyServer(server); - if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal'); - expect(cancelRemoval(removal.deploymentId)).toBe(true); - expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' }); - }); - - it('preserves stop intent across process restarts, including an uncertain stop request', async () => { - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const id = await queue('stop'); - finishDeployment(id, { ok: false, message: 'lost response' }); - expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' }); - // Exercise a legacy upgrade where the stop exists but intent was not stored. - db().prepare("UPDATE servers SET desired_state = 'running'").run(); - db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run(); - closeDb(); - expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' }); - const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js'); - await incident(); - expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' }); - await queue('start'); - expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' }); - }); - - it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => { - const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const result = requestWatchdogLifecycle(server, 'restart', 'incident'); - expect(result.ok).toBe(true); - if (!result.ok) throw new Error(result.error); - expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result); - expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server, - label: 'Server down', href: '/maintenance/incident' }]); - expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' }); - db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run(); - await processProvisioning(); - expect(komodoRestart).not.toHaveBeenCalled(); - db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run(); - await processProvisioning(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => { - const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { config } = await import('../../src/lib/server/config.js'); - vi.spyOn(config.rcon, 'password', 'get').mockReturnValue(''); - vi.mocked(rconGetPlayerCount).mockResolvedValue(null); - vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null }); - await incident(); await report(0); - const result = requestWatchdogLifecycle(server, 'restart', 'incident'); - if (!result.ok) throw new Error(result.error); - await processProvisioning(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.setSystemTime(Date.now() + 15_000); - await processProvisioning(); - expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' }); - await report(0); - vi.setSystemTime(Date.now() + 15_000); - await processProvisioning(); - expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' }); - }); - - it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => { - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const id = await queue('restart'); - vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; }); - await processProvisioning(); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 }); - await report(0); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3); - await processProvisioning(); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 }); - }); - - it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => { - const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValue(null); - for (const offset of [-90_001, 1]) { - await report(0, new Date(Date.now() + offset).toISOString()); - expect(telemetryEvidence(server)).toBeNull(); - expect(await probePlayerCount(server)).toBeNull(); - } - await report(0); - expect(telemetryEvidence(server, new Date().toISOString())).toBeNull(); - db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server); - expect(telemetryEvidence(server)).toBeNull(); - }); - - it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => { - const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - await incident(); - const restart = requestWatchdogLifecycle(server, 'restart', 'incident'); - if (!restart.ok) throw new Error(restart.error); - await processProvisioning(); - expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false }); - vi.setSystemTime(Date.now() + 300_000); - const rebuild = escalateWatchdogRestart('incident', restart.deploymentId); - if (!rebuild.ok) throw new Error(rebuild.error); - expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild); - expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' }); - expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ operation_id: rebuild.deploymentId, healthy_at: null }); - closeDb(); - vi.mocked(rconGetPlayerCount).mockResolvedValue(null); - vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null }); - await processProvisioning(); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' }); - expect(komodoRestart).toHaveBeenCalledTimes(1); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' }); - await processProvisioning(); - expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' }); - expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined(); - }); - - it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => { - const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const restart = requestWatchdogLifecycle(server, 'restart', 'incident'); - if (!restart.ok) throw new Error(restart.error); - await processProvisioning(); vi.setSystemTime(Date.now() + 300_000); - removeServerKey(server); - expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false }); - expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' }); - expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId }); - saveServerKey(server, 'saved-key'); - db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server); - expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false }); - expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' }); - expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 }); - }); - - it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => { - const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const restart = requestWatchdogLifecycle(server, 'restart', 'incident'); - if (!restart.ok) throw new Error(restart.error); - await processProvisioning(); vi.setSystemTime(Date.now() + 300_000); - let reportHealthy!: (value: { ok: true; state: string }) => void; - vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; })); - const observation = refreshRestartRollout(); - const rebuild = escalateWatchdogRestart('incident', restart.deploymentId); - if (!rebuild.ok) throw new Error(rebuild.error); - reportHealthy({ ok: true, state: 'running' }); - await observation; - expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ operation_id: rebuild.deploymentId, healthy_at: null }); - expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' }); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' }); - }); - - it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => { - const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js'); - const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident'); - if (!rebuild.ok) throw new Error(rebuild.error); - deploymentStage(rebuild.deploymentId, REMOVING_SERVER); - finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' }); - db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run(); - db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run(); - db().prepare('DELETE FROM servers WHERE name = ?').run(server); - expect(resumeRebuild(rebuild.deploymentId)).toBe(true); - await processProvisioning(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' }); - expect(komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - - it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => { - const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js'); - const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const restart = requestWatchdogLifecycle(server, 'restart', 'incident'); - if (!restart.ok) throw new Error(restart.error); - await processProvisioning(); vi.setSystemTime(Date.now() + 300_000); - const rebuild = escalateWatchdogRestart('incident', restart.deploymentId); - if (!rebuild.ok) throw new Error(rebuild.error); - db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)") - .run(rebuild.deploymentId, new Date().toISOString()); - vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' }); - await processProvisioning(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' }); - db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run(); - const retry = requestWatchdogLifecycle(server, action, 'incident'); - if (!retry.ok) throw new Error(retry.error); - expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ operation_id: retry.deploymentId, healthy_at: null }); - await refreshRestartRollout(); - // Old process health cannot release capacity before this retry submits. - expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null }); - if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' }); - await processProvisioning(); - expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' }); - expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1); - }); - - it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => { - const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await incident(); - const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident'); - if (!rebuild.ok) throw new Error(rebuild.error); - const now = new Date().toISOString(); - db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now); - db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId); - db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server); - db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server); - db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' }); - await processProvisioning(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' }); - expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId }); - expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 }); - expect(db().pragma('foreign_key_check')).toEqual([]); - await processProvisioning(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' }); - vi.setSystemTime(Date.now() + 15_000); await report(0); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null }); - await processProvisioning(); - expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' }); - }); -}); - -describe('cancelling submitted legacy commands', () => { - it.each(['start', 'restart', 'stop'] as const)('ignores a late %s response after cancellation', async action => { - const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const komodo = await import('../../src/lib/server/integrations/komodo.js'); - let release!: (value: { ok: true }) => void; - const command = action === 'restart' ? komodo.komodoRestart : komodo.komodoRuntimeAction; - vi.mocked(command).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); - const id = await queue(action); - const pending = processProvisioning(); - await vi.waitFor(() => expect(release).toBeTypeOf('function')); - expect(cancelProvisioningJob(id)).toBe(true); - release({ ok: true }); - await pending; - expect(await operation(id)).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' }); - expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(id)).toEqual([]); - const replacement = await queue(action); - await processProvisioning(); - expect(await operation(replacement)).toMatchObject({ state: 'awaiting-health' }); - expect(command).toHaveBeenCalledTimes(2); - }); -}); diff --git a/tests/unit/rcon-status.test.ts b/tests/unit/rcon-status.test.ts index 37bf216..00ee33f 100644 --- a/tests/unit/rcon-status.test.ts +++ b/tests/unit/rcon-status.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; import { createSocket, type Socket } from 'node:dgram'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js'; +import { rconConfigured } from '../../src/lib/server/config.js'; import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js'; import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js'; import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js'; @@ -10,11 +10,11 @@ import { nativeInstance, nativeRequest } from '../../src/lib/server/backends/nat vi.mock('node:dgram', () => ({ createSocket: vi.fn() })); vi.mock('../../src/lib/server/config.js', () => ({ config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } }, - komodoConfigured: vi.fn(), rconConfigured: vi.fn() + rconConfigured: vi.fn() })); vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() })); vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() })); -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() })); + vi.mock('../../src/lib/server/backends/native.js', () => ({ nativeInstance: vi.fn(), nativeRequest: vi.fn() })); class FakeSocket extends EventEmitter { @@ -34,7 +34,6 @@ beforeEach(() => { server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow; vi.mocked(getServer).mockImplementation(() => ({ ...server })); vi.mocked(latestGameSubmission).mockReturnValue(null); - vi.mocked(komodoConfigured).mockReturnValue(false); vi.mocked(rconConfigured).mockReturnValue(true); vi.mocked(createSocket).mockImplementation(() => { const socket = new FakeSocket(); diff --git a/tests/unit/restart-rollout.test.ts b/tests/unit/restart-rollout.test.ts deleted file mode 100644 index 8c59e23..0000000 --- a/tests/unit/restart-rollout.test.ts +++ /dev/null @@ -1,494 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; - -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() -})); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ - rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() -})); - -let fixture: Fixture; -const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3']; -const healthyRcon = { - ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '', - status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } -}; - -beforeEach(async () => { - vi.resetModules(); - vi.resetAllMocks(); - vi.useFakeTimers({ toFake: ['Date'] }); - vi.setSystemTime(new Date('2026-09-17T15:00:00Z')); - fixture = await newFixture(servers); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test'); - vi.stubEnv('KOMODO_API_SECRET', 'test'); - vi.stubEnv('RCON_PASSWORD', 'test'); - vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); - const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); - await seedServerRegistry(); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(komodoRestart).mockResolvedValue({ ok: true }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rconGetPlayerCount).mockResolvedValue(0); - vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon); -}); - -afterEach(async () => { - await fixture.cleanup(); - vi.unstubAllEnvs(); - vi.useRealTimers(); -}); - -function nextTick() { vi.setSystemTime(Date.now() + 15_000); } - -it('ignores a late restart reply after deletion without changing a replacement recovery slot', async () => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); - const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); - const { restartInRollout } = await import('../../src/lib/server/provisioning/restarts.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - let complete!: (value: { ok: true }) => void; - vi.mocked(komodoRestart).mockImplementationOnce(() => new Promise(resolve => { complete = resolve; })); - const oldOperation = beginDeployment(servers[0], 'restart'); - const pending = restartInRollout(servers[0], { operationId: oldOperation, action: 'restart' }); - db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); - await seedServerRegistry(); - const replacement = beginDeployment(servers[0], 'restart'); - const requestedAt = new Date().toISOString(); - db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') - .run(servers[0], requestedAt, replacement); - nextTick(); - complete({ ok: true }); - expect(await pending).toMatchObject({ kind: 'failed' }); - expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE id = ?').get(oldOperation)) - .toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) }); - expect(db().prepare('SELECT operation_id, requested_at, error FROM restart_rollout WHERE server = ?').get(servers[0])) - .toEqual({ operation_id: replacement, requested_at: requestedAt, error: null }); -}); - -async function deployment(id = 'dep-1', targets = servers, action = 'promote') { - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)') - .run(id, action, new Date().toISOString()); - for (const server of targets) { - db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) - VALUES (?, ?, 1, 'pending')`).run(id, server); - } -} - -async function rows(id = 'dep-1') { - const { db } = await import('../../src/lib/server/db/index.js'); - return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server') - .all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>; -} - -async function beta(targets: string[], waitForHealth = true) { - const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') }); - return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }], - servers: targets, waitForHealth, waitForMatch: false }); -} - -describe('rolling deployment restarts', () => { - it('restarts one server per recovery and finishes only after the last server is healthy', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect((await sweep()).restarted).toBe(1); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); - expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]); - await sweep(); // Repeated kick can't bypass the grace period. - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 }); - expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']); - nextTick(); - expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 }); - nextTick(); - expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 }); - expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true); - expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server])); - expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1')) - .toMatchObject({ finished_at: expect.any(String) }); - }); - - it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => { - await deployment('z-oldest', [servers[2]], 'rollback'); - nextTick(); - await deployment('a-next', [servers[0]], 'drift'); - nextTick(); - await deployment('b-latest', [servers[1]], 'promote'); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]); - nextTick(); - await sweep(); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]); - }); - - it('skips occupied and unreachable servers without consuming capacity', async () => { - await deployment(); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]); - }); - - it('keeps the fleet queued while either container or RCON health is failing', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - await sweep(); - nextTick(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' }); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - }); - - it('coalesces overlapping sweeps while the remote restart is unresolved', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - let finish!: () => void; - const entered = new Promise((resolve) => { - vi.mocked(komodoRestart).mockImplementationOnce(() => { - resolve(); - return new Promise((done) => { finish = () => done({ ok: true }); }); - }); - }); - const first = sweep(); - await entered; - const second = sweep(); - expect(second).toBe(first); - // Even a long-running restart request cannot be freed by a health poll. - nextTick(); - const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js'); - await refreshRestartRollout(); - expect(rolloutWaitReason(servers[1])).toMatch(/Queued/); - finish(); - await Promise.all([first, second]); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('deduplicates simultaneous deployment rows for the same server', async () => { - await deployment('dep-1', [servers[0]]); - await deployment('dep-2', [servers[0], servers[1]]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - expect((await rows('dep-1'))[0].restarted_at).toBeTruthy(); - expect((await rows('dep-2'))[0].restarted_at).toBeTruthy(); - }); - - it('does not count a later deployment as having been applied by an earlier restart', async () => { - await deployment('dep-1', [servers[0]]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - await deployment('dep-2', [servers[0]]); - nextTick(); - await sweep(); - expect((await rows('dep-1'))[0].restart_state).toBe('done'); - expect((await rows('dep-2'))[0].restart_state).toBe('pending'); - // Provisioning can submit the next job in the pass that verifies the - // first, but it must issue a distinct restart and await its recovery. - expect(komodoRestart).toHaveBeenCalledTimes(2); - expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - expect((await rows('dep-2'))[0].restarted_at).toBeTruthy(); - }); - - it('preserves occupied slots across app/module restarts', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - await sweep(); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); - vi.resetModules(); - const fresh = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await fresh.sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - await fresh.sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - }); - - it('adopts pre-upgrade restarts instead of starting a second server', async () => { - await deployment(); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?') - .run(new Date().toISOString(), servers[0]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('retains capacity after a timeout because the remote restart may have been accepted', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' }); - await sweep(); - expect((await rows())[0].restart_state).toBe('failed'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' }); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('does not restart a server whose file deployment failed', async () => { - await deployment(); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?") - .run(servers[0]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]); - expect((await rows())[0].error).toBe('write failed'); - }); - - it('releases slots for definite local failures that could not have started a restart', async () => { - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' }); - await sweep(); - expect((await rows())[0].restart_state).toBe('failed'); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]); - }); -}); - -describe('beta and deployment coordination', () => { - it('limits beta restarts and releases capacity only after recovery', async () => { - await beta(servers); - const { betaTick } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const first = betaTick(); - expect(betaTick()).toBe(first); - await first; - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - await betaTick(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - }); - - it('shares the same capacity when beta and deployment ticks run concurrently', async () => { - await beta([servers[1]]); - await deployment('dep-1', [servers[0]]); - const { betaTick } = await import('../helpers/provisioning.js'); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await Promise.all([sweep(), betaTick()]); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('retains beta recovery slots even when the beta skips health checks or is closed', async () => { - const { betaId } = await beta([servers[0]], false); - const { betaTick } = await import('../helpers/provisioning.js'); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - await betaTick(); - expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' }); - db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId); - await deployment('dep-1', [servers[1]]); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(1); - nextTick(); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - }); - - it('checks player occupancy again when resuming a legacy restarting beta', async () => { - const { betaId } = await beta([servers[0]]); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?") - .run(new Date(Date.now() - 30_000).toISOString(), betaId); - const { betaTick } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValue(3); - await betaTick(); - expect(komodoRestart).not.toHaveBeenCalled(); - }); -}); - -describe('Provisioning ownership', () => { - it('persists caller requests before probes and completes them without another caller tick', async () => { - await deployment('campaign', [servers[0]]); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - await sweep(); await sweep(); - const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string }; - const id = source.provisioning_id; - expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null }); - expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0], - label: 'Promotion campaign', href: '/campaigns/campaign' }]); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(komodoInspectHealthy).not.toHaveBeenCalled(); - expect(rconGetPlayerCount).not.toHaveBeenCalled(); - expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 }); - await processProvisioning(); nextTick(); await processProvisioning(); - expect(provisioningOperation(id)?.state).toBe('done'); - expect((await rows('campaign'))[0].restart_state).toBe('pending'); - await sweep(); - expect((await rows('campaign'))[0].restart_state).toBe('done'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('lets concurrent callers submit while only the owner admits restart capacity', async () => { - await deployment('campaign', [servers[0]]); await beta([servers[1]]); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - await Promise.all([sweep(), betaTick()]); - expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled(); - const first = processProvisioning(); - expect(processProvisioning()).toBe(first); - await first; - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('does not replay an ambiguous campaign restart after recovery or app restart', async () => { - await deployment('campaign', [servers[0]]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' }); - await sweep(); nextTick(); await sweep(); - const { closeDb } = await import('../../src/lib/server/db/index.js'); - closeDb(); vi.resetModules(); - const fresh = await import('../helpers/provisioning.js'); - nextTick(); await fresh.sweep(); - expect((await rows('campaign'))[0].restart_state).toBe('failed'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('holds a paused beta request and cancels closed work without a beta runner tick', async () => { - const { betaId } = await beta([servers[0]]); - const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); - await betaTick(); - const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - await processProvisioning(); - expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' }); - expect(rconGetPlayerCount).not.toHaveBeenCalled(); - db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId); - await processProvisioning(); - expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' }); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => { - await deployment('campaign', [servers[0]]); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' }); - await sweep(); nextTick(); await sweep(); - const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js'); - expect(requestLifecycle(servers[0], 'restart').ok).toBe(true); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - expect((await rows('campaign'))[0].restart_state).toBe('failed'); - }); - - it('does not credit a legacy submission with an older completed recovery slot', async () => { - await deployment('campaign', [servers[0]]); - const { db } = await import('../../src/lib/server/db/index.js'); - const old = new Date().toISOString(); nextTick(); - const submitted = new Date().toISOString(); - db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old); - db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' }); - expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null }); - nextTick(); await sweep(); - expect((await rows('campaign'))[0].restart_state).toBe('done'); - expect(komodoRestart).not.toHaveBeenCalled(); - }); - - it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => { - const { betaId } = await beta([servers[0]]); - const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); - await betaTick(); - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string }; - closeDb(); vi.resetModules(); - const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js'); - const source = { kind: 'beta' as const, id: String(betaId) }; - expect(requestRestart(servers[0], source).id).toBe(previous.id); - releaseRestartRequest(servers[0], source); - const next = requestRestart(servers[0], source); - expect(next.id).not.toBe(previous.id); - expect(provisioningOperation(previous.id)?.state).toBe('cancelled'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(komodoRestart).toHaveBeenCalledTimes(1); - expect(provisioningOperation(next.id)?.state).toBe('awaiting-health'); - }); -}); - -describe('rollout configuration', () => { - it('honors a configured limit larger than one', async () => { - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 }); - await deployment(); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - }); - - it('holds new restarts after lowering a live limit without releasing active slots', async () => { - const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); - const { sweep } = await import('../helpers/provisioning.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - saveFleetSettings({ ...fleetSettings(), restartConcurrency: 2 }); - await deployment(); - await sweep(); - saveFleetSettings({ ...fleetSettings(), restartConcurrency: 1 }); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(2); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all()).toHaveLength(2); - nextTick(); - await sweep(); - expect(komodoRestart).toHaveBeenCalledTimes(3); - }); -}); diff --git a/tests/unit/rollback-staging.test.ts b/tests/unit/rollback-staging.test.ts index a782dc3..8430498 100644 --- a/tests/unit/rollback-staging.test.ts +++ b/tests/unit/rollback-staging.test.ts @@ -98,7 +98,7 @@ describe('deferred promotion rollback', () => { expect(existsSync(join(dirname(path), 'never.gsc'))).toBe(false); }); - it('keeps another campaign eligible when withdrawing a shared activation source', async () => { + it('keeps another campaign eligible when withdrawing an independent activation source', async () => { const { path, before } = await promotion('script', true); const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); @@ -109,7 +109,7 @@ describe('deferred promotion rollback', () => { queueFileChange({ sourceKind: 'campaign', sourceId: 'other', server, kind: 'script', relPath: 'other.gsc', content: Buffer.from('other campaign'), inventorySource: 'drift:other' }); const shared = requestRestart(server, { kind: 'campaign', id: 'promote' }); - expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).toBe(shared.id); + expect(requestRestart(server, { kind: 'campaign', id: 'other' }).id).not.toBe(shared.id); await rollbackPromoteDeployment('promote'); await applyCampaign('other'); expect(await readFile(path)).toEqual(before); diff --git a/tests/unit/script-flow.test.ts b/tests/unit/script-flow.test.ts index 770aba9..043d390 100644 --- a/tests/unit/script-flow.test.ts +++ b/tests/unit/script-flow.test.ts @@ -9,7 +9,7 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']); - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; }); diff --git a/tests/unit/script-upload.test.ts b/tests/unit/script-upload.test.ts index e328930..a4cb035 100644 --- a/tests/unit/script-upload.test.ts +++ b/tests/unit/script-upload.test.ts @@ -4,7 +4,7 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); - delete process.env.KOMODO_ADDRESS; delete process.env.RCON_PASSWORD; + delete process.env.RCON_PASSWORD; }); afterEach(async () => { await fixture.cleanup(); }); function request(content: string, reuse = false) { diff --git a/tests/unit/server-chat.test.ts b/tests/unit/server-chat.test.ts index f38f7dc..81c0f91 100644 --- a/tests/unit/server-chat.test.ts +++ b/tests/unit/server-chat.test.ts @@ -94,7 +94,7 @@ describe('server chat API', () => { it('rejects registered names that escape the server directory', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run('..', '..'); + db().prepare('INSERT INTO servers (name) VALUES (?)').run('..'); expect((await get('..')).status).toBe(400); }); @@ -130,8 +130,8 @@ describe('server chat API', () => { await report('heartbeat', 40, 8); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('DELETE FROM servers WHERE name = ?').run(SERVER); - db().prepare(`INSERT INTO servers (name, komodo_name, game, map, created_at) - VALUES (?, ?, 't5', 'zombie_theater', ?)`).run(SERVER, SERVER, at(70)); + db().prepare(`INSERT INTO servers (name, game, map, created_at) + VALUES (?, 't5', 'zombie_theater', ?)`).run(SERVER, at(70)); await report('game_start', 80); await report('heartbeat', 90, 2); await log(['60:50 say;abc;0;Player;previous server', '61:40 say;abc;0;Player;current server'], 100); diff --git a/tests/unit/server-detail-api.test.ts b/tests/unit/server-detail-api.test.ts index 43002cf..1c79be3 100644 --- a/tests/unit/server-detail-api.test.ts +++ b/tests/unit/server-detail-api.test.ts @@ -9,7 +9,7 @@ describe('server detail data', () => { it('filters history before the global limit and includes the matching campaign members', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const handle = db(); - handle.exec("INSERT INTO servers (name, komodo_name) VALUES ('target', 'target'), ('other', 'other')"); + handle.exec("INSERT INTO servers (name) VALUES ('target'), ('other')"); handle.transaction(() => { const deployment = handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES (?, 'promote', ?)"); const member = handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'done')"); @@ -32,7 +32,7 @@ describe('server detail data', () => { }); it('identifies plugins, scripts, composed configs and unknown inventory files', async () => { const { db } = await import('../../src/lib/server/db/index.js'); - db().exec("INSERT INTO servers (name, komodo_name) VALUES ('target', 'target')"); + db().exec("INSERT INTO servers (name) VALUES ('target')"); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); const { version } = createScript({ name:'Plugin', kind:'plugin', targetRelPath:'native.bin', content:Buffer.from('plugin') }); diff --git a/tests/unit/server-history-storage.test.ts b/tests/unit/server-history-storage.test.ts index 64c5b6c..aee18f3 100644 --- a/tests/unit/server-history-storage.test.ts +++ b/tests/unit/server-history-storage.test.ts @@ -20,7 +20,7 @@ function database(legacy = false) { } function seedHistory(db: Database.Database) { - db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'server', 28960); + db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960); INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'script', 'test.gsc'); INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 4, X'74657374'); INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'promoted'); @@ -78,7 +78,7 @@ describe('durable server history', () => { expect(db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()).toEqual({ restart_state: 'done', restarted_at: 'restarted' }); expect(db.prepare('SELECT content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') }); expect(db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) }); - db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'replacement', 28960); + db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960); INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new'); INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('replacement', 'server', 'health', 'open', 'New incident', 'new', 'new');`); migrateServerHistory(db); @@ -134,7 +134,7 @@ describe('durable server history', () => { ('superseded', 'removed', 'rebuild', 'failed', 'Removing server files and registry', 'before', 'ui'), ('destroyed', 'removed', 'destroy', 'done', 'Removing server files and registry', 'after', 'ui'); INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('superseded'); - INSERT INTO servers (name, komodo_name, destroyed_at) VALUES ('tombstone', 'tombstone', 'old'); + INSERT INTO servers (name, destroyed_at) VALUES ('tombstone', 'old'); INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('gone', 'queued', 'old'), ('tombstone', 'done', 'old'); INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('old', 'Old collection', 'old', '["tombstone","gone","new"]');`); migrateServerHistory(db); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts index 9dde4c0..de8b522 100644 --- a/tests/unit/server-status.test.ts +++ b/tests/unit/server-status.test.ts @@ -4,9 +4,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn() -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn() })); @@ -21,12 +18,13 @@ afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); - db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run(); - const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + db().prepare("INSERT INTO servers (name, last_error) VALUES ('managed', 'timeout:old process')").run(); + vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth'); + const { nativeHealth } = await import('../../src/lib/server/backends/native.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); - return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer }; + return { db: db(), inspect: vi.mocked(nativeHealth), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer }; } describe('database fleet and container readiness', () => { @@ -152,7 +150,9 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); const now = Date.parse('2026-10-02T12:00:00Z'); vi.setSystemTime(now); - for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`); + for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name) VALUES (?)').run(`fast-${i}`); + const { prepareNativeHost } = await import('../helpers/native-host.js'); + await prepareNativeHost(['managed', ...Array.from({ length: 9 }, (_, i) => `fast-${i}`)]); let release!: (result: { ok: true; state: string }) => void; let active = 0; let peak = 0; @@ -191,7 +191,7 @@ describe('database fleet and container readiness', () => { } }); - it.each(['native', 'komodo'])('gives each %s host two slots through inspection and RCON, independently of a stalled host', async backend => { + it('gives each host two slots through inspection and RCON, independently of a stalled host', async () => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); db.prepare("DELETE FROM servers WHERE name='managed'").run(); const hosts = await import('../../src/lib/server/hosts/index.js'); @@ -200,9 +200,9 @@ describe('database fleet and container readiness', () => { hosts.enrollHost(hosts.createEnrollment(registered.id).token, { protocolVersion: 1 }); for (let instance = 0; instance < 4; instance++) { const name = `host-${host}-${instance}`; - db.prepare('INSERT INTO servers (name, komodo_name, komodo_server) VALUES (?, ?, ?)') - .run(name, name, backend === 'komodo' ? registered.id : null); - if (backend === 'native') db.prepare(`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at) + db.prepare('INSERT INTO servers (name) VALUES (?)') + .run(name); + db.prepare(`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at) VALUES (?, ?, ?, 'library', 'image', ?, ?)`).run(name, name, registered.id, 28960 + instance, new Date().toISOString()); } } @@ -254,7 +254,7 @@ describe('database fleet and container readiness', () => { vi.useFakeTimers({ toFake: ['Date'] }); const now = Date.parse('2026-10-02T12:00:00Z'); vi.setSystemTime(now); - db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run(); + db.prepare("INSERT INTO servers (name) VALUES ('slow')").run(); let release!: (result: { ok: true; state: string }) => void; inspect.mockImplementation(async name => name === 'slow' ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) @@ -531,7 +531,7 @@ describe('database fleet and container readiness', () => { expect(listServers()).toEqual([]); }); - it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => { + it('skips startup RCON, clears old errors, and resumes polling only after the host agent reports readiness', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' }); diff --git a/tests/unit/staged-files.test.ts b/tests/unit/staged-files.test.ts index 9a34e73..09f50e9 100644 --- a/tests/unit/staged-files.test.ts +++ b/tests/unit/staged-files.test.ts @@ -9,9 +9,6 @@ vi.mock('node:fs', async (importOriginal) => { return { ...original, writeFileSync: vi.fn(original.writeFileSync) }; }); -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); @@ -32,17 +29,12 @@ beforeEach(async () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-27T15:00:00Z')); fixture = await newFixture(servers); - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); - vi.stubEnv('KOMODO_API_KEY', 'test'); - vi.stubEnv('KOMODO_API_SECRET', 'test'); + vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); - const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(komodoRestart).mockResolvedValue({ ok: true }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', @@ -84,9 +76,13 @@ async function queueChange(source: Source, targets = [servers[0]]) { return { sourceId, inventory, replacement }; } +// File-storage unit tests exercise activation directly. Native provisioning and +// safety suites cover admission, host delivery, player gates, and recovery. async function tick(source: Source) { - const { sweep, betaTick } = await import('../helpers/provisioning.js'); - await (source === 'campaign' ? sweep() : betaTick()); + const { db } = await import('../../src/lib/server/db/index.js'); + const { activateFiles } = await import('../helpers/staged-files.js'); + const sources = db().prepare('SELECT DISTINCT source_id FROM staged_file_changes WHERE source_kind=? AND state=\'pending\'').all(source) as { source_id: string }[]; + for (const row of sources) await activateFiles(source, row.source_id); } async function expectLiveUnchanged(inventory: Awaited>['inventory'], server = servers[0]) { @@ -95,96 +91,7 @@ async function expectLiveUnchanged(inventory: Awaited(['campaign', 'beta'])('%s staged file activation', (source) => { - it.each(blockers)('preserves live bytes and inventory while waiting for %s', async (blocker) => { - const { db } = await import('../../src/lib/server/db/index.js'); - const { config } = await import('../../src/lib/server/config.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - if (blocker === 'occupied') vi.mocked(rconGetPlayerCount).mockResolvedValue(3); - if (blocker === 'unknown occupancy') vi.mocked(rconGetPlayerCount).mockResolvedValue(null); - if (blocker === 'missing Komodo') Object.assign(config.komodo, { address: '' }); - if (blocker === 'missing RCON') Object.assign(config.rcon, { password: '' }); - if (blocker === 'previous restart' || blocker === 'rollout capacity') { - db().prepare('INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)') - .run(blocker === 'previous restart' ? servers[0] : servers[1], new Date().toISOString()); - } - const { inventory, replacement } = await queueChange(source); - await expectLiveUnchanged(inventory); - await tick(source); - await expectLiveUnchanged(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - - // The replacement and its inventory must already be installed when - // Komodo sees the activation request, after every admission gate passes. - Object.assign(config.komodo, { address: 'http://komodo.test' }); - Object.assign(config.rcon, { password: 'test' }); - vi.mocked(rconGetPlayerCount).mockResolvedValue(0); - db().prepare('DELETE FROM restart_rollout').run(); - let bytesAtRestart: Buffer | undefined; - let inventoryAtRestart: typeof inventory; - vi.mocked(komodoRestart).mockImplementationOnce(async () => { - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - bytesAtRestart = await readFile(destination()); - inventoryAtRestart = getServerScript(servers[0], relPath); - return { ok: true }; - }); - await tick(source); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); - expect(bytesAtRestart).toEqual(after); - expect(inventoryAtRestart).toMatchObject({ - script_version_id: replacement.id, sha256: replacement.sha256, size: after.length - }); - }); - - it('does not write when its provisioning request is cancelled during the occupancy probe', async () => { - const { inventory } = await queueChange(source); - const { db } = await import('../../src/lib/server/db/index.js'); - const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { - const operation = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND state = 'waiting'") - .get(servers[0]) as { id: string }; - cancelProvisioning(operation.id); - return 0; - }); - await tick(source); - await expectLiveUnchanged(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT state FROM server_deployments WHERE server = ?').get(servers[0])) - .toEqual({ state: 'cancelled' }); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - }); - - it('never applies old staged work to a replacement server created during its occupancy probe', async () => { - await queueChange(source); - const { db } = await import('../../src/lib/server/db/index.js'); - const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); - const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const replacementBytes = Buffer.from('new server instance'); - let replacementInventory: ReturnType; - vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { - db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]); - await seedServerRegistry(); - await writeFile(destination(), replacementBytes); - recordScript(servers[0], relPath, null, sha256Buffer(replacementBytes), replacementBytes.length, 'replacement'); - replacementInventory = getServerScript(servers[0], relPath); - return 0; - }); - await tick(source); - expect(await readFile(destination())).toEqual(replacementBytes); - expect(getServerScript(servers[0], relPath)).toEqual(replacementInventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE server = ?').get(servers[0])) - .toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) }); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - }); it('preserves existing file permissions when activating staged bytes', async () => { await queueChange(source); @@ -204,260 +111,13 @@ describe.each(['campaign', 'beta'])('%s staged file activation', (source }); it('keeps the queued payload across an application restart', async () => { - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(rconGetPlayerCount).mockResolvedValue(2); const { inventory } = await queueChange(source); - await tick(source); await expectLiveUnchanged(inventory); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); vi.resetModules(); + await expectLiveUnchanged(inventory); await tick(source); - await expectLiveUnchanged(inventory); - const freshRcon = await import('../../src/lib/server/integrations/rcon.js'); - vi.mocked(freshRcon.rconGetPlayerCount).mockResolvedValue(0); - await tick(source); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); expect(await readFile(destination())).toEqual(after); }); - - it('does not restart or retain rollout capacity when installing staged files fails', async () => { - const { inventory } = await queueChange(source); - const { db } = await import('../../src/lib/server/db/index.js'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - // A directory appearing at the destination causes a deterministic I/O - // failure even when the test process can bypass ordinary file modes. - await rm(destination()); - await mkdir(destination()); - await tick(source); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(getServerScript(servers[0], relPath)).toEqual(inventory); - expect(db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL').all()).toEqual([]); - expect(db().prepare('SELECT state, error FROM server_deployments WHERE server = ?').get(servers[0])) - .toMatchObject({ state: 'failed', error: expect.any(String) }); - }); -}); - -describe('staged source eligibility', () => { - it.each(['held', 'done'])('does not apply a campaign changed to %s after it requests provisioning', async (state) => { - const { sourceId, inventory } = await queueChange('campaign'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await sweep(); - db().prepare('UPDATE deployment_servers SET restart_state = ? WHERE deployment_id = ?').run(state, sourceId); - await processProvisioning(); - await expectLiveUnchanged(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - }); - - it('does not install a beta withdrawn after it requests provisioning', async () => { - const { sourceId, inventory } = await queueChange('beta'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await betaTick(); - db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(sourceId); - await processProvisioning(); - await expectLiveUnchanged(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - }); -}); - - -describe('staged drift removals and review guards', () => { - it('defers removing a file and its inventory until the empty-server restart', async () => { - const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); - const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); - const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await writeFile(destination(), before); - recordScript(servers[0], relPath, null, sha256Buffer(before), before.length, 'fixture'); - const inventory = getServerScript(servers[0], relPath); - vi.mocked(rconGetPlayerCount).mockResolvedValue(1); - await runDriftDeployment({ servers: [servers[0]] }); - await expectLiveUnchanged(inventory); - await tick('campaign'); - await expectLiveUnchanged(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - vi.mocked(rconGetPlayerCount).mockResolvedValue(0); - let fileExistedAtRestart: boolean | undefined; - let inventoryAtRestart: typeof inventory; - vi.mocked(komodoRestart).mockImplementationOnce(async () => { - fileExistedAtRestart = existsSync(destination()); - inventoryAtRestart = getServerScript(servers[0], relPath); - return { ok: true }; - }); - await tick('campaign'); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); - expect(fileExistedAtRestart).toBe(false); - expect(inventoryAtRestart).toBeUndefined(); - }); - - it('fails without overwriting a reviewed target changed while its restart was queued', async () => { - const { inventory } = await queueChange('campaign'); - const externalChange = Buffer.from('changed outside Electric Cherry while queued'); - await writeFile(destination(), externalChange); - await tick('campaign'); - const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - expect(await readFile(destination())).toEqual(externalChange); - expect(getServerScript(servers[0], relPath)).toEqual(inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - expect(db().prepare('SELECT error FROM server_deployments WHERE server = ?').get(servers[0])) - .toMatchObject({ error: expect.stringContaining('changed while queued') }); - }); -}); - - -describe('shared activation and partial writes', () => { - it('keeps coalesced campaigns unchanged while one source is held, then activates both once resumed', async () => { - const first = await queueChange('campaign'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const second = await runDriftDeployment({ servers: [servers[0]] }); - await sweep(); - const linked = db().prepare('SELECT provisioning_id FROM deployment_servers ORDER BY deployment_id').all() as { provisioning_id: string }[]; - expect(linked).toHaveLength(2); - expect(new Set(linked.map(row => row.provisioning_id)).size).toBe(1); - db().prepare("UPDATE deployment_servers SET restart_state = 'held' WHERE deployment_id = ?").run(second.deploymentId); - await processProvisioning(); - await expectLiveUnchanged(first.inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'waiting' }); - db().prepare("UPDATE deployment_servers SET restart_state = 'pending' WHERE deployment_id = ?").run(second.deploymentId); - await tick('campaign'); - expect(await readFile(destination())).toEqual(after); - expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); - expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all()) - .toEqual([{ state: 'applied' }, { state: 'applied' }]); - }); - - it('rejects a beta admission while an earlier file campaign remains queued and unapplied', async () => { - const campaign = await queueChange('campaign'); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - await sweep(); - const { uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const version = uploadNewVersion(campaign.replacement.script_id, Buffer.from('new beta bytes')); - await expect(openBeta({ scripts: [{ scriptId: version.script_id, scriptVersionId: version.id }], - servers: [servers[0]], waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' }); - await expectLiveUnchanged(campaign.inventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT * FROM betas').all()).toEqual([]); - expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign.sourceId)) - .toEqual({ restart_state: 'pending' }); - expect(db().prepare("SELECT state FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(campaign.sourceId)) - .toEqual({ state: 'pending' }); - }); - - it('lets a beta rollback restore its files behind a legacy overlapping drift campaign', async () => { - const beta = await queueChange('beta'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); - const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - // Recreate overlapping work retained from before the admission guard. - // The beta was already open; this older drift still expects its original - // live bytes and must fail independently of the later restoration. - db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('legacy-drift', 'drift', ?)") - .run(new Date(Date.now() - 60_000).toISOString()); - db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) - VALUES ('legacy-drift', ?, 1, 'pending')`).run(servers[0]); - queueFileChange({ sourceKind: 'campaign', sourceId: 'legacy-drift', server: servers[0], kind: 'script', - relPath, content: Buffer.from('older queued drift bytes'), inventorySource: 'drift:legacy-drift', - expectedBeforeSha: sha256Buffer(before) }); - await sweep(); - await tick('beta'); - expect(await readFile(destination())).toEqual(after); - const rollback = await rollbackBeta(Number(beta.sourceId)); - vi.setSystemTime(Date.now() + 15_000); - await tick('campaign'); - expect(await readFile(destination())).toEqual(before); - expect(komodoRestart).toHaveBeenCalledTimes(2); - const old = db().prepare('SELECT provisioning_id, restart_state, error FROM deployment_servers WHERE deployment_id = ?') - .get('legacy-drift') as { provisioning_id: string; restart_state: string; error: string }; - const restoration = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?') - .get(rollback.deploymentId) as { provisioning_id: string }; - expect(old).toMatchObject({ restart_state: 'failed', error: expect.stringContaining('Superseded by rollback') }); - expect(restoration.provisioning_id).not.toBe(old.provisioning_id); - }); - - it('records partial installation without restarting and never automatically retries the failed write', async () => { - const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); - const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { recordScript, getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); - const { sha256Buffer } = await import('../../src/lib/server/fleet/files.js'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - const first = createScript({ name: 'First in bundle', targetRelPath: relPath, content: after }); - const second = createScript({ name: 'Second in bundle', targetRelPath: 'second.gsc', content: after }); - const secondPath = join(fixture.t5, servers[0], 'runtime/plutonium/storage/t5/raw/scripts/sp/second.gsc'); - for (const [path, relative] of [[destination(), relPath], [secondPath, 'second.gsc']]) { - await writeFile(path, before); - recordScript(servers[0], relative, null, sha256Buffer(before), before.length, 'fixture'); - } - const secondInventory = getServerScript(servers[0], 'second.gsc'); - await openBeta({ scripts: [first, second].map(({ script, version }) => ({ scriptId: script.id, scriptVersionId: version.id })), - servers: [servers[0]], waitForHealth: false, waitForMatch: false }); - const actualFs = await vi.importActual('node:fs'); - vi.mocked(writeFileSync).mockImplementation((path, content, options) => { - if (path === secondPath) throw new Error('Simulated disk write failure'); - actualFs.writeFileSync(path, content, options); - }); - await tick('beta'); - expect(await readFile(destination())).toEqual(after); - expect(getServerScript(servers[0], relPath)?.script_version_id).toBe(first.version.id); - expect(await readFile(secondPath)).toEqual(before); - expect(getServerScript(servers[0], 'second.gsc')).toEqual(secondInventory); - expect(komodoRestart).not.toHaveBeenCalled(); - expect(db().prepare('SELECT state FROM staged_file_changes ORDER BY id').all()) - .toEqual([{ state: 'applied' }, { state: 'applying' }]); - expect(db().prepare('SELECT state, submitted_at, error FROM server_deployments').get()) - .toMatchObject({ state: 'failed', submitted_at: null, error: expect.stringContaining('Simulated disk write failure') }); - expect(db().prepare('SELECT * FROM restart_rollout').all()).toEqual([]); - const attempts = vi.mocked(writeFileSync).mock.calls.length; - await tick('beta'); - expect(writeFileSync).toHaveBeenCalledTimes(attempts); - expect(komodoRestart).not.toHaveBeenCalled(); - }); -}); - -describe('rollback supersedes provisioning', () => { - it.each(['when-empty', 'manual'] as const)('replaces an unhealthy beta restart with a %s rollback without waiting for recovery', async restart => { - const beta = await queueChange('beta'); - const { db } = await import('../../src/lib/server/db/index.js'); - const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js'); - const { komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); - await tick('beta'); - const old = db().prepare('SELECT provisioning_id FROM beta_servers WHERE beta_id=?').get(beta.sourceId) as { provisioning_id: string }; - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'awaiting-health' }); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'container failed to start' }); - const rollback = await rollbackBeta(Number(beta.sourceId), { restart }); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(old.provisioning_id)).toEqual({ state: 'cancelled' }); - expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(old.provisioning_id)).toEqual([]); - const replacement = db().prepare('SELECT provisioning_id,restart_state FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as { provisioning_id: string; restart_state: string }; - expect(replacement.provisioning_id).not.toBe(old.provisioning_id); - await tick('campaign'); - expect(await readFile(destination())).toEqual(restart === 'manual' ? after : before); - expect(komodoRestart).toHaveBeenCalledTimes(restart === 'manual' ? 1 : 2); - expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(replacement.provisioning_id)) - .toEqual({ state: restart === 'manual' ? 'waiting' : 'awaiting-health' }); - }); }); diff --git a/tests/unit/sweep.test.ts b/tests/unit/sweep.test.ts index d52c4c8..ab0ffc8 100644 --- a/tests/unit/sweep.test.ts +++ b/tests/unit/sweep.test.ts @@ -7,7 +7,7 @@ beforeEach(async () => { vi.resetModules(); fixture = await newFixture(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']); // Provisioning must retain queued work while integrations are unset. - delete process.env.KOMODO_ADDRESS; + delete process.env.RCON_PASSWORD; }); @@ -95,7 +95,7 @@ describe('sweep', () => { for (const r of rows) expect(r.restart_state).toBe('pending'); const operations = db().prepare('SELECT state, stage FROM server_deployments').all(); expect(operations).toHaveLength(2); - for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting for Komodo and RCON configuration' }); + for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting to verify the server is empty' }); }); it('is a no-op when nothing is pending', async () => { diff --git a/tests/unit/system-status.test.ts b/tests/unit/system-status.test.ts index 292b079..7a87567 100644 --- a/tests/unit/system-status.test.ts +++ b/tests/unit/system-status.test.ts @@ -5,46 +5,33 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); - vi.stubEnv('KOMODO_ADDRESS', ''); - vi.stubEnv('KOMODO_API_KEY', ''); - vi.stubEnv('KOMODO_API_SECRET', ''); - vi.stubEnv('KOMODO_TARGET_SERVER', ''); - vi.stubEnv('RCON_PASSWORD', ''); + vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret'); }); -afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); -it('directs a fresh controller to native hosts and verified libraries without probing Komodo', async () => { - const probe = vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe'); +it('directs a fresh controller to host enrollment and verified game files', async () => { const { GET } = await import('../../src/routes/api/system-status/+server.js'); - const response = await GET({ url: new URL('http://localhost/api/system-status') }); + const response = await GET(); const body = await response.json(); expect(response.headers.get('cache-control')).toBe('no-store'); - expect(body.checks.find((check: { id: string }) => check.id === 'hosts').state).toBe('unconfigured'); - expect(body.checks.find((check: { id: string }) => check.id === 'game-files').state).toBe('attention'); - expect(body.checks.some((check: { id: string }) => check.id === 'komodo')).toBe(false); + expect(body.checks.map((check: { id: string }) => check.id)).toEqual(['hosts', 'runtime', 'game-files', 'rcon', 'keys', 'first-server']); + expect(body.checks[0]).toMatchObject({ state: 'unconfigured', href: '/hosts' }); + expect(body.checks[2].state).toBe('attention'); expect(body.readyServers).toBe(0); - expect(probe).not.toHaveBeenCalled(); + expect(JSON.stringify(body)).not.toContain('fixture-rcon-secret'); }); -it('never exposes credentials or claims that network reachability validates them', async () => { - vi.stubEnv('KOMODO_ADDRESS', 'https://komodo.example.test'); - vi.stubEnv('KOMODO_API_KEY', 'fixture-private-key'); - vi.stubEnv('KOMODO_API_SECRET', 'fixture-private-secret'); - vi.stubEnv('RCON_PASSWORD', 'fixture-rcon-secret'); - vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 443, resolvedIp: '192.0.2.1', outcome: 'connected', elapsedMs: 1 }); +it('reports an enrolled but disconnected host as needing attention without issuing commands', async () => { + const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const host = createHost({ name: 'Game host' }); + const enrolled = enrollHost(createEnrollment(host.id).token, { protocolVersion: 1 }); + db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', host.id); const { GET } = await import('../../src/routes/api/system-status/+server.js'); - const text = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).text(); - for (const secret of ['fixture-private-key', 'fixture-private-secret', 'fixture-rcon-secret']) expect(text).not.toContain(secret); - const connection = JSON.parse(text).checks.find((check: { id: string }) => check.id === 'komodo'); - expect(connection.state).toBe('ready'); - expect(connection.detail).toContain('does not validate API credentials'); -}); - -it('reports a configured unreachable host as needing attention', async () => { - vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.example.test:9120'); - vi.stubEnv('KOMODO_API_KEY', 'key'); vi.stubEnv('KOMODO_API_SECRET', 'secret'); - vi.spyOn(await import('../../src/lib/server/integrations/netProbe.js'), 'tcpProbe').mockResolvedValue({ label: 'Komodo', host: 'komodo.example.test', port: 9120, resolvedIp: null, outcome: 'dns-fail', elapsedMs: 1 }); - const { GET } = await import('../../src/routes/api/system-status/+server.js'); - const body = await (await GET({ url: new URL('http://localhost/api/system-status?probe=1') })).json(); - expect(body.checks.find((check: { id: string }) => check.id === 'komodo').state).toBe('attention'); + const response = await GET(); + const text = await response.text(); + const body = JSON.parse(text); + expect(body.checks[0]).toMatchObject({ state: 'attention', detail: expect.stringContaining('0 connected') }); + expect(text).not.toContain(enrolled.credential); + expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]); }); diff --git a/tests/unit/t6-config-flow.test.ts b/tests/unit/t6-config-flow.test.ts index 6b69d38..5c8dc45 100644 --- a/tests/unit/t6-config-flow.test.ts +++ b/tests/unit/t6-config-flow.test.ts @@ -14,7 +14,7 @@ const cfgPath = (server: string) => join(fixture.t6, server, 'configs/dedicated_ beforeEach(async () => { vi.resetModules(); fixture = await newFixture([town, farm, t5]); - vi.stubEnv('KOMODO_ADDRESS', ''); + vi.stubEnv('RCON_PASSWORD', ''); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); diff --git a/tests/unit/ui-server-clock.test.ts b/tests/unit/ui-server-clock.test.ts index e1cf500..bd0f698 100644 --- a/tests/unit/ui-server-clock.test.ts +++ b/tests/unit/ui-server-clock.test.ts @@ -210,7 +210,7 @@ describe('Fleet report clock', () => { const { fleetServers, hydrateFleet } = await setup(); await vi.advanceTimersByTimeAsync(2000); emit(epoch + 2000, [status(epoch + 2000, { players: 3 })]); - const integrations = { rcon: true, komodo: false, rconTimeoutMs: 3000, defaultRconHost: '' }; + const integrations = { rcon: true, rconTimeoutMs: 3000, defaultRconHost: '' }; hydrateFleet({ serverNow: epoch + 1000, servers: [server(epoch + 1000), server(epoch, { name: 'two' })], operations: [], integrations }); expect(get(fleetServers)).toHaveLength(2); expect(get(fleetServers)[0].last_players).toBe(3); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index 14cecb1..37b6240 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -1,14 +1,10 @@ +import { prepareNativeHost } from '../helpers/native-host.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { gameEvent } from '../helpers/telemetry.js'; -vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ - komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(), - komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(), - komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn() -})); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); @@ -20,7 +16,8 @@ let fixture: Fixture; type Engine = typeof import('../../src/lib/server/watchdog/index.js'); let watchdog: Engine; let database: typeof import('../../src/lib/server/db/index.js'); -let komodo: typeof import('../../src/lib/server/integrations/komodo.js'); +let backend: typeof import('../../src/lib/server/backends/native.js'); +let refreshHost: () => unknown; let rcon: typeof import('../../src/lib/server/integrations/rcon.js'); beforeEach(async () => { @@ -29,22 +26,18 @@ beforeEach(async () => { vi.setSystemTime(new Date('2026-09-27T12:00:00Z')); fixture = await newFixture([server]); vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); - for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test', - KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1', - SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value); + for (const [key, value] of Object.entries({ RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1' })) vi.stubEnv(key, value); database = await import('../../src/lib/server/db/index.js'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server); const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); saveServerKey(server, 'saved-key'); - komodo = await import('../../src/lib/server/integrations/komodo.js'); - for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) { - vi.mocked(komodo[name]).mockResolvedValue({ ok: true }); - } - vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name })); - vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true }); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + refreshHost = (await prepareNativeHost([server])).heartbeat; + backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' }); + const { mockAgentFiles } = await import('../helpers/agent-files.js'); + await mockAgentFiles(); rcon = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0); vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon); @@ -56,8 +49,8 @@ afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAl const incidents = () => watchdog.watchdogOverview().incidents; const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; }; const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[]; -function advance(ms: number) { vi.setSystemTime(Date.now() + ms); } -function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); } +function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); } +function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); } async function pollStatus() { const worker = await vi.importActual('../../src/lib/server/pipeline/worker.js'); await worker.fanoutStatus(); @@ -78,7 +71,7 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(backend.nativeHealth).not.toHaveBeenCalled(); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); expect(incidents()).toEqual([]); expect(operations()).toEqual([]); @@ -98,7 +91,7 @@ describe('Watchdog detection and ownership', () => { } expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' }); expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]); - expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(3); + expect(backend.nativeHealth).toHaveBeenCalledTimes(3); expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0); }); @@ -113,7 +106,7 @@ describe('Watchdog detection and ownership', () => { advance(60_000); await watchdog.watchdogTick(); expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at }); - expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1); + expect(backend.nativeHealth).toHaveBeenCalledTimes(1); advance(90_001); await pollStatus(); await watchdog.watchdogTick(); @@ -205,7 +198,7 @@ describe('Watchdog detection and ownership', () => { await watchdog.watchdogTick(); } expect(scan).toHaveBeenCalledTimes(1); - expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(5); + expect(backend.nativeHealth).toHaveBeenCalledTimes(5); advance(watchdog.WATCHDOG_SCAN_MS); await pollStatus(); await watchdog.watchdogTick(); @@ -256,32 +249,11 @@ describe('Watchdog detection and ownership', () => { expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]); await watchdog.watchdogTick(true); expect(operations()).toHaveLength(1); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); + expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]); }); - it('keeps paused recovery observational and resumes one pending incident', async () => { - watchdog.setWatchdogEnabled(false); - down(); - for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(onlyIncident()).toMatchObject({ state: 'open', failures: 4 }); - expect(operations()).toEqual([]); - database.closeDb(); - expect(watchdog.watchdogOverview().enabled).toBe(false); - watchdog.setWatchdogEnabled(true); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - watchdog.setWatchdogEnabled(false); - await processProvisioning(); - expect(operations()[0].state).toBe('waiting'); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); - watchdog.setWatchdogEnabled(true); - await processProvisioning(); - expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); - }); - - it('does not restart an unknown Komodo state even after repeated sustained observations', async () => { - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo request timed out' }); + it('does not restart an unknown host agent state even after repeated sustained observations', async () => { + vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' }); for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' }); expect(operations()).toEqual([]); @@ -297,7 +269,7 @@ describe('Watchdog detection and ownership', () => { }); it('allows a longer initial startup grace period before requesting recovery', async () => { - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); + vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } expect(operations()).toEqual([]); await watchdog.watchdogTick(true); @@ -346,79 +318,6 @@ describe('Watchdog detection and ownership', () => { expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); }); - it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => { - const row = await sustainedFailure(); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); - }); - - it('holds queued health recovery when container readiness becomes unknown', async () => { - await sustainedFailure(); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' }); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(operations()[0].state).toBe('waiting'); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); - }); - - it('does not apply a recovered occupancy probe after its operation is replaced', async () => { - const row = await sustainedFailure(); - const restart = operations()[0]; - const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => { - expect(cancelProvisioning(restart.id)).toBe(true); - expect(requestLifecycle(server, 'stop').ok).toBe(true); - return 0; - }); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering'); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }), - expect.objectContaining({ action: 'stop', state: 'waiting' })]); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); - }); - - it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => { - const row = await sustainedFailure(); - const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); - const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)') - .run(server, new Date().toISOString()).lastInsertRowid; - const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) }); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }), - expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]); - expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); - }); - - it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => { - await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - if (action === 'rebuild') { - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - await watchdog.watchdogTick(true); - } - const queued = operations().at(-1)!; - const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id }); - expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' }); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; }); - await processProvisioning(); - expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' }); - if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); - else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1); - }); - it('preserves intentional stop intent after completion and database reopening', async () => { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const stop = requestLifecycle(server, 'stop'); @@ -430,7 +329,7 @@ describe('Watchdog detection and ownership', () => { for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } expect(incidents()).toEqual([]); expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]); - expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(backend.nativeHealth).not.toHaveBeenCalled(); }); it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => { @@ -447,12 +346,12 @@ describe('Watchdog detection and ownership', () => { for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } expect(incidents()).toEqual([]); expect(operations()).toEqual([]); - expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(backend.nativeHealth).not.toHaveBeenCalled(); }); it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => { + vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => { expect(requestLifecycle(server, 'stop').ok).toBe(true); return { ok: false, state: 'exited', reason: 'Container exited' }; }); @@ -477,7 +376,7 @@ describe('Watchdog detection and ownership', () => { advance(watchdog.WATCHDOG_SCAN_MS); } expect(incidents()).toEqual([]); - expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); + expect(backend.nativeHealth).not.toHaveBeenCalled(); expect(rcon.rconStatusResult).not.toHaveBeenCalled(); expect(operations()).toHaveLength(1); @@ -495,7 +394,7 @@ describe('Watchdog detection and ownership', () => { it('stops probing when provisioning takes ownership during the container inspection', async () => { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); - vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => { + vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => { expect(requestLifecycle(server, 'restart').ok).toBe(true); return { ok: true, state: 'running' }; }); @@ -507,173 +406,6 @@ describe('Watchdog detection and ownership', () => { }); describe('Watchdog recovery persistence', () => { - it('verifies recovery at the deadline before deciding whether to rebuild', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ healthy_at: new Date().toISOString() }); - }); - - it('defers escalation while current container health is unknown', async () => { - await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' }); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]); - }); - - it('keeps failed submission history when fresh evidence later releases recovery', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' }); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ healthy_at: new Date().toISOString() }); - expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost'); - }); - - it('discards escalation evidence if another lifecycle submission appears during its probe', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => { - database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) - VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString()); - return healthyRcon; - }); - await watchdog.watchdogTick(true); - expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' }); - expect(operations().some(op => op.action === 'rebuild')).toBe(false); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering'); - expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null }); - }); - - it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - await watchdog.watchdogTick(true); - expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); - const rebuild = operations()[1]; - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - await processProvisioning(); - expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server)) - .toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() }); - expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - - it('preserves the original finding as deleted history after an intentional successful destroy', async () => { - const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); - const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js'); - removeServerKey(server); - await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); - await watchdog.watchdogTick(true); - const id = onlyIncident().id; - expect(onlyIncident().state).toBe('attention'); - expect(destroyServer(server)).toMatchObject({ ok: true }); - down(); - await processDestructions(); - expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]); - expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined(); - await watchdog.watchdogTick(true); - expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'attention', server_deleted_at: expect.any(String) }); - expect(watchdog.updateWatchdogIncident(id, 'retry')).toMatchObject({ ok: false, status: 409 }); - }); - - it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); - expect(operations()[0].state).toBe('awaiting-health'); - advance(watchdog.WATCHDOG_RECOVERY_MS - 1); - await watchdog.watchdogTick(true); - expect(operations()).toHaveLength(1); - advance(1); - await watchdog.watchdogTick(true); - expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); - expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(2); - vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' }); - await processProvisioning(); - expect(operations()[1].state).toBe('failed'); - await watchdog.watchdogTick(true); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention'); - database.closeDb(); - for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); } - expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention'); - expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); - expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1); - }); - - it('does not escalate a timed out restart while automatic recovery is paused', async () => { - await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - watchdog.setWatchdogEnabled(false); - advance(watchdog.WATCHDOG_RECOVERY_MS); - await watchdog.watchdogTick(true); - expect(operations()).toHaveLength(1); - watchdog.setWatchdogEnabled(true); - await watchdog.watchdogTick(true); - expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); - }); - - it('reclaims a failed attempt recovery slot only after an explicit incident retry', async () => { - const row = await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - await watchdog.watchdogTick(true); - vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' }); - await processProvisioning(); - await watchdog.watchdogTick(true); - const failed = operations()[1]; - expect(failed).toMatchObject({ action: 'rebuild', state: 'failed' }); - expect(onlyIncident()).toMatchObject({ id: row.id, state: 'attention', cycle: 1 }); - const slot = () => database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server); - expect(slot()).toEqual({ operation_id: failed.id, healthy_at: null }); - - await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected'); - vi.mocked(komodo.komodoInspectHealthy).mockClear(); - advance(watchdog.WATCHDOG_RECOVERY_MS); - await watchdog.watchdogTick(true); - expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled(); - expect(operations()).toHaveLength(2); - - expect(watchdog.updateWatchdogIncident(row.id, 'retry')).toEqual({ ok: true }); - await watchdog.watchdogTick(true); - expect(operations()).toHaveLength(2); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); - const retry = operations()[2]; - expect(retry).toMatchObject({ action: 'restart', state: 'waiting' }); - expect(onlyIncident()).toMatchObject({ id: row.id, state: 'recovering', cycle: 2 }); - expect(slot()).toEqual({ operation_id: retry.id, healthy_at: null }); - expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(3); - - await processProvisioning(); - expect(komodo.komodoRestart).toHaveBeenCalledTimes(2); - expect(operations()[2].state).toBe('awaiting-health'); - }); it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => { down(); @@ -770,41 +502,6 @@ describe('Watchdog APIs', () => { }); describe('Watchdog drift campaigns', () => { - it('queues an audited rebuild without changing files until provisioning executes and retains its campaign through cleanup', async () => { - await mkdir(join(fixture.t5, server, 'configs'), { recursive: true }); - const path = join(fixture.t5, server, 'configs', 'stray.cfg'); - await writeFile(path, 'unexpected content'); - await watchdog.watchdogTick(true); - const row = onlyIncident(); - expect(row).toMatchObject({ kind: 'drift', state: 'recovering' }); - const action = watchdog.watchdogDetail(row.id)!.actions[0]; - expect(action).toMatchObject({ action: 'rebuild', campaign_id: expect.any(String), operation_state: 'waiting' }); - expect(await readFile(path, 'utf8')).toBe('unexpected content'); - expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled(); - expect(database.db().prepare('SELECT rel_path, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(action.campaign_id)) - .toEqual([{ rel_path: 'stray.cfg', content_before: Buffer.from('unexpected content'), content_after: null }]); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - const { processCreations } = await import('../../src/lib/server/provisioning/instances.js'); - const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); - down(); - await processProvisioning(); - expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'queued' })]); - await expect(access(path)).rejects.toThrow(); - expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined(); - database.closeDb(); - await sweep(); await watchdog.watchdogTick(true); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering'); - expect(database.db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)) - .toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' }); - await processCreations(); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - advance(1); - await processCreations(); await sweep(); await watchdog.watchdogTick(true); - expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); - expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' }); - expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'done' })]); - expect(komodo.komodoRestart).not.toHaveBeenCalled(); - }); it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => { const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); @@ -812,43 +509,10 @@ describe('Watchdog drift campaigns', () => { const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'); await writeFile(path, 'unexpected'); await watchdog.watchdogTick(true); - expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('Save a server key') }); + expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('server key are required') }); advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); expect(operations()).toEqual([]); expect(await readFile(path, 'utf8')).toBe('unexpected'); - expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled(); - }); -}); - -describe('Maintenance recovery policy boundaries', () => { - it('separates health recovery from drift repair and holds an already queued drift rebuild', async () => { - const policies = await import('../../src/lib/server/maintenance/policies.js'); - policies.setMaintenancePolicy('health', false); - down(); - for (let n = 0; n < 3; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } - expect(operations()).toEqual([]); - expect(onlyIncident().kind).toBe('health'); - vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); - await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'drift'); - await watchdog.watchdogTick(true); - expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'waiting' })]); - policies.setMaintenancePolicy('drift', false); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - expect(operations()[0].state).toBe('waiting'); - expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled(); - }); - it('global pause blocks escalation without disabling the saved health policy', async () => { - await sustainedFailure(); - const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); - await processProvisioning(); - const policies = await import('../../src/lib/server/maintenance/policies.js'); - policies.pauseMaintenance(true); - advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); - expect(operations()).toHaveLength(1); - expect(policies.maintenancePolicies().health).toBe(true); - policies.pauseMaintenance(false); await watchdog.watchdogTick(true); - expect(operations()).toHaveLength(2); - expect(operations()[1].action).toBe('rebuild'); + expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]); }); });