Finish autoscaling diagnostics and document pilot rollout

README.md

@@ -152,6 +152,8 @@ affect occupancy or game history. Existing JSON payloads are unchanged.
 - [Native hosts, storage and operation recovery](docs/native-hosts.md)
 - [Users, roles and authentication](docs/authentication.md)
 - [Provisioning and safety queues](docs/provisioning.md)
+- [Lobby autoscaling implementation plan](docs/lobby-autoscaling-plan.md)
+- [Lobby autoscaling and pilot procedure](docs/autoscaling.md)
 - [Collections and promoted launch recipes](docs/deployment-blueprints.md)
 - [Maintenance](docs/maintenance.md)
 - [Beta test plans and game telemetry](docs/beta-test-plans.md)

agent/ec_agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.6.0"
+VERSION = "1.7.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -71,6 +71,10 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+class AutoscalingInvalidated(Rejected):
+    """This removal needs a new policy decision, never a later local retry."""
+
+
 def systemctl(*args):
     try:
         result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -1356,7 +1360,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1581,6 +1585,24 @@ class Agent:
             raise AgentError("Staged file checksum mismatch")
         return content
 
+    def authorize_autoscaling_removal(self, instance_id, generation, payload):
+        authorization = payload.get("autoscaling")
+        if not isinstance(authorization, dict) or not self.executing_id or payload.get("safetyOverride") or payload.get("stageId"):
+            raise AutoscalingInvalidated("Invalid autoscaling removal authorization")
+        started = time.monotonic()
+        try:
+            response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(self.executing_id, safe="") + "/authorize-removal", authorization)
+        except AgentError:
+            raise AutoscalingInvalidated("Autoscaling removal authorization unavailable; request a new decision") from None
+        ttl = response.get("validForMs") if isinstance(response, dict) else None
+        if not isinstance(response, dict) or response.get("authorized") is not True or isinstance(ttl, bool) or not isinstance(ttl, (int, float)) or not 0 < ttl <= 5000:
+            raise AutoscalingInvalidated("Autoscaling removal authorization was withdrawn or expired")
+        # Occupancy can change during the controller round trip. Check again
+        # within the short claim window, immediately before the durable checkpoint.
+        self.safe_to_disrupt(instance_id, generation)
+        if (time.monotonic() - started) * 1000 >= ttl:
+            raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
+
     def lifecycle(self, kind, instance_id, generation, payload):
         if kind == "instance.cancel-create":
             return self.cancel_creation(instance_id, generation, payload)
@@ -1610,6 +1632,8 @@ class Agent:
             # The authenticated controller records explicit administrator approval
             # for this immutable operation. Ownership and generation checks still apply.
             override_safety = payload.get("safetyOverride") is True
+            if "autoscaling" in payload and (kind != "instance.remove" or override_safety):
+                raise AutoscalingInvalidated("Autoscaling only permits removal with player safety checks")
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
@@ -1618,6 +1642,8 @@ class Agent:
                 self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation)
+            if "autoscaling" in payload:
+                self.authorize_autoscaling_removal(instance_id, generation, payload)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
                 before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
@@ -2203,7 +2229,11 @@ class Agent:
         try:
             result = self.dispatch(operation)
         except Held as err:
+            if operation.get("kind") == "instance.remove" and "autoscaling" in operation.get("payload", {}):
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
             return self.journal.update(operation["id"], "accepted", {"blocked": "occupied" if "Players" in str(err) else "unknown", "message": str(err)})
+        except AutoscalingInvalidated as err:
+            return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
         except Exception as err:
             # The command may already have changed Docker or files. Never infer
             # failure means nothing happened, and never retry without inspection.
@@ -2239,6 +2269,8 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":
                 return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
             if kind == "instance.create":

agent/tests/test_agent.py

@@ -959,6 +959,61 @@ class AgentTests(unittest.TestCase):
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}
         self.assertEqual(self.agent.execute(operation)["state"], "completed")
 
+    def autoscaling_operation(self, op_id="autoscaling-remove"):
+        return self.operation("instance.remove", {"autoscaling": {"policyId": "policy", "revision": 1, "expiresAt": 2000000}}, op_id)
+
+    def test_autoscaling_occupied_or_unknown_removal_never_retries_later(self):
+        self.create()
+        for index, players in enumerate((2, None)):
+            operation = self.autoscaling_operation(f"autoscaling-{index}")
+            self.agent.rcon = lambda *_, count=players: {"ok": True, "players": count}
+            with patch.object(self.agent, "request", side_effect=AssertionError("Occupied removal requested authorization")):
+                result = self.agent.execute(operation)
+                self.assertEqual(result["state"], "failed")
+                self.assertTrue(json.loads(result["result"])["autoscalingInvalidated"])
+                self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+                self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertFalse(any(args[:2] == ("container", "stop") for args in self.docker.calls))
+
+    def test_autoscaling_removal_requires_fresh_one_shot_controller_authorization(self):
+        self.create()
+        self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+        with patch.object(self.agent, "request", return_value={"authorized": False}) as request:
+            self.assertEqual(self.agent.execute(self.autoscaling_operation("revoked"))["state"], "failed")
+            self.assertTrue(request.call_args.args[1].endswith("/revoked/authorize-removal"))
+        with patch.object(self.agent, "request", return_value={"authorized": True, "validForMs": 5000}):
+            self.assertEqual(self.agent.execute(self.autoscaling_operation("valid"))["state"], "completed")
+        self.assertEqual(sum(args[:2] == ("container", "rm") for args in self.docker.calls), 1)
+
+    def test_autoscaling_rechecks_players_after_authorization_and_rejects_expired_claim(self):
+        self.create()
+        with patch.object(self.agent, "rcon", side_effect=[{"ok": True, "players": 0}, {"ok": True, "players": 1}]), patch.object(self.agent, "request", return_value={"authorized": True, "validForMs": 5000}):
+            self.assertEqual(self.agent.execute(self.autoscaling_operation("joined"))["state"], "failed")
+        self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+        with patch.object(self.agent, "request", return_value={"authorized": True, "validForMs": 5000}), patch.object(module.time, "monotonic", side_effect=[0, 6]):
+            self.assertEqual(self.agent.execute(self.autoscaling_operation("expired"))["state"], "failed")
+        self.assertFalse(any(args[:2] == ("container", "stop") for args in self.docker.calls))
+
+    def test_autoscaling_restart_before_mutation_invalidates_instead_of_replaying(self):
+        self.create()
+        operation = self.autoscaling_operation()
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "running")
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        with patch.object(self.agent, "request", side_effect=AssertionError("Recovered command requested execution")):
+            result = self.agent.execute(operation)
+        self.assertEqual(result["state"], "failed")
+        self.assertTrue(json.loads(result["result"])["autoscalingInvalidated"])
+        self.assertFalse(any(args[:2] == ("container", "stop") for args in self.docker.calls))
+
+    def test_autoscaling_cannot_use_admin_override(self):
+        self.create()
+        operation = self.autoscaling_operation()
+        operation["payload"]["safetyOverride"] = True
+        self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        self.assertFalse(any(args[:2] == ("container", "stop") for args in self.docker.calls))
+
     def test_occupancy_hold_explains_local_rcon_failure_without_exposing_reply(self):
         self.create()
         operation = self.operation("instance.restart")

docs/autoscaling.md

@@ -0,0 +1,64 @@
+# Lobby autoscaling
+
+Open **Fleet → Autoscaling** to keep empty lobbies available for new groups. Policies apply to a specific game and map variant. A partly occupied Zombies lobby is not an empty spare, even when it still has player slots.
+
+Autoscaling is opt-in. Installing the feature creates no policies and changes no lobby counts. Use a two-week dry run before enabling live actions.
+
+## Configure a policy
+
+Choose **Add policy**, select the game and map, and enter an explicit maximum. The initial editor selects Dry run, with a minimum of one running lobby, one empty spare, a 20-minute empty interval, and a five-minute action cooldown. Counts must be between 1 and 20; the minimum and spare target cannot exceed the maximum.
+
+Choose **Preview changes** to inspect the existing lobbies, saved key coverage, target, proposed action, and blockers. **Save reviewed policy** checks the policy revision and current fleet again. If the preview changed, review a new preview before saving.
+
+| Mode | Behavior |
+| --- | --- |
+| Off | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled. |
+| Dry run | Records hypothetical decisions without reserving identities, hosts, or ports. Collections retain control of the lineup. |
+| Create only | Adds a lobby when eligible. Surplus removal remains advisory. |
+| Create and destroy | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. |
+
+Live activation requires saved Plutonium keys for enough exact lobby identities to support the configured maximum. Keys may be nonconsecutive: if only lobby 3 and lobby 8 are keyed, those identities can be used. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host.
+
+The desired count is `min(maximum, max(minimum, occupied lobbies + empty spare target))`. The controller evaluates policies every 15 seconds and requests at most one unresolved operation per map. Create uses the existing startup concurrency, CPU, memory, library, port, and host admission checks. Lack of host capacity leaves one request waiting; it does not create additional requests every tick. Creates use current promoted launch files when provisioning begins.
+
+## Ownership and blockers
+
+A live policy owns the map's lobby count. Competing collection launches and manual Create, Start, Stop, Rebuild, or Destroy requests are rejected with a policy link. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish.
+
+Unknown or stale occupancy, stopped or unhealthy members, active betas, campaigns, maintenance, host drains, and unresolved recovery hold the policy's decisions. Autoscaling does not repair these conditions itself. The page and linked Provisioning operation explain the current blocker, including capacity and missing-key waits.
+
+Current positive occupancy from either telemetry or RCON vetoes an empty claim. Empty intervals reset when players arrive, evidence becomes unknown, the instance or lifecycle changes, or observation continuity breaks. Controller downtime does not count toward the empty duration. Failed or uncertain automatic operations pause further actions until an operator inspects their Provisioning recovery and reviews the policy again.
+
+## Pause and cancellation
+
+Use **Pause all** to stop new decisions globally, or **Edit or pause → Off** for one map. A global pause retains each live policy's ownership; setting it Off releases ownership once its outstanding operation settles. Resuming the global switch preserves each policy's selected mode.
+
+Unsubmitted creates and queued removals are withdrawn where possible. Cancelling an autoscaling operation from Provisioning also turns its policy Off so it cannot immediately request a replacement.
+
+An automatic removal requires fresh controller eligibility, a short-lived authorization, and a host-local occupancy check. The agent claims authorization once, repeats its occupancy check, and must begin mutation within five seconds of requesting the claim. Occupied, unknown, expired, interrupted-before-mutation, or revoked attempts end without waiting to destroy the lobby later. A new attempt needs a new decision and another complete empty interval. Automatic removals cannot use the administrator safety override.
+
+The authorization claim is the cancellation boundary. A pause after that point cannot guarantee withdrawal; the page retains the outstanding operation until its receipt is reconciled. Do not treat cancellation as proof that a delivered host command had no effect. Uncertain commands retain their identity and resource reservations.
+
+Turning a policy Off does not recreate lobbies it removed or remove lobbies it created. Once operations settle, use Collections to restore a fixed lineup through the normal capacity and empty-server gates.
+
+## Dry run and pilot
+
+The production observation period and live pilot have not been run as part of implementation. Begin them on the operator-selected maps after deploying the controller and compatible agents.
+
+1. Select one or two representative maps and save policies in Dry run for two weeks. Record the baseline lineup, keys, versions, host CPU and memory, player-hours, and comparable days of the week.
+2. Review recent hypothetical actions, empty lobby-hours, minutes below the spare target, unknown-status minutes, capacity/key blockers, and projected action frequency. Dry run observes the actual lineup; it cannot predict how players would redistribute after a removal.
+3. Enable Create only on the pilot maps after resolving blockers. Measure request-to-ready time from Provisioning, browser visibility, and successful external joins separately. Increase the spare target if creation latency leaves new groups without a suitable lobby.
+4. Agree the acceptable spare-deficit duration for the pilot, verify agent capability, and enable Create and destroy. Keep normal gameplay and joins under observation while comparing actual host CPU and empty lobby-hours with the baseline.
+5. Expand one map at a time only when savings are measurable and spare availability and gameplay meet the agreed tolerance. Pause immediately if those conditions regress.
+
+The page retains up to 31 days of decision evidence and observations, with the 30 most recent decision changes shown. Repeated unchanged evaluations update the decision's last-seen time instead of adding a row. Unresolved intent survives retention. Provisioning history retains the operation and policy backlink separately.
+
+Measurement totals exclude collection gaps. Request-to-verified-readiness summaries include queue time and only completed creates; they do not establish browser visibility or successful joins. “Time with players” in Metrics can rise simply because idle lobbies leave its denominator. Use host CPU and memory measurements alongside player-hours and spare availability when assessing savings.
+
+## API and implementation
+
+`GET /api/autoscaling` returns policies, current evidence, recent decisions, measurements, and global pause state. Viewers may read it; mutations require an operator or administrator, using the existing authentication and origin checks.
+
+`POST /api/autoscaling` accepts `preview`, `save`, or `global-pause`. Preview/save requests contain `settings`, plus `id` and `revision` when editing. Save also requires the preview `token`. Global pause uses a boolean `paused` field. Agent removal claims use a separate host-authenticated endpoint and are bound to the exact command, host, instance generation, policy revision, and expiry.
+
+Shared contracts live in `src/lib/autoscaling.ts`; the evaluator, snapshots, persistence, ownership, and authorization live in `src/lib/server/autoscaling/`. Provisioning remains the only lifecycle executor. See the [approved implementation plan](lobby-autoscaling-plan.md) for the design and rollout criteria.

docs/lobby-autoscaling-plan.md

@@ -0,0 +1,148 @@
+# Lobby autoscaling implementation plan
+
+Status: Approved and implemented October 6, 2026. Controller support and agent 1.7.0 are complete; production dry run, pilot measurements, and live rollout remain pending. See [usage and the pilot procedure](autoscaling.md).
+
+Introduce an opt-in policy for each game and map variant that keeps a small number of empty, ready lobbies available. When the last spare fills, request another lobby through Provisioning. Remove surplus lobbies only after sustained, confirmed inactivity. This reduces idle hosting while preserving room for new groups, including when existing Zombies sessions have free player slots but are already well into a game.
+
+Start with Create/Destroy on existing native hosts. Run the evaluator in dry run for two weeks, then enable creation for a small pilot before enabling removal. Host acquisition, VM shutdown, Stop/Start pooling, scheduled demand prediction, and waking servers from incoming packets are outside the first release.
+
+## Existing foundations and required extensions
+
+| Foundation | Reuse and required work |
+| --- | --- |
+| Background worker | Add a requester to [pipeline/worker.ts](../src/lib/server/pipeline/worker.ts), which already reconciles background work every 15 seconds independently of Provisioning. Keep host execution inside Provisioning. |
+| Occupancy evidence | Reuse [status evidence](../src/lib/server/fleet/status-evidence.ts) and shared freshness rules. Monitoring runs on a 30-second cadence; accepted evidence currently expires after 90 seconds. Positive occupancy vetoes an empty claim. Unknown is never zero. |
+| Safe creation | Use the durable creation intent and placement path in [provisioning/native.ts](../src/lib/server/provisioning/native.ts). Preserve CPU, memory, port, host eligibility, library, and startup concurrency checks. A blocked create remains queued. |
+| Safe removal | Reuse the controller occupancy checks, instance generation checks, host-local check, and cleanup checkpoints. Add autoscaling-specific admission and cancellation behavior before enabling automatic destruction. |
+| Requester eligibility | [provisioning/worker.ts](../src/lib/server/provisioning/worker.ts) currently gives Create and Destroy separate paths that bypass the runtime requester eligibility check. Autoscaling ownership and policy checks must cover both paths, including delayed execution. A source label alone is insufficient. |
+| Saved keys | [server-keys.ts](../src/lib/server/fleet/server-keys.ts) stores keys by exact lobby name and accepts lobby numbers 1–20. Allocate only unused identities with saved keys; never copy a key to a different identity. |
+| Collections | [collections.ts](../src/lib/server/fleet/collections.ts) already counts queued creations and launches promoted recipes. Exact matching can remove lobbies across the fleet, including maps omitted from the collection. Add ownership checks to preview and submission. |
+| Hosting insights | [metrics-insights.ts](../src/lib/metrics-insights.ts) provides historical suggestions for hosting trials. Use these to help choose policies; the live evaluator needs current empty-lobby evidence and must not execute historical suggestions directly. |
+
+Create already uses the promoted launch recipe and capacity-aware placement. Start retains existing files and has recovery concurrency controls, but would need equivalent CPU and memory admission before becoming an autoscaling strategy. The proposed scope follows option 4 in [the idle CPU research](t5-t6-idle-cpu-research.md).
+
+## Policy and ownership
+
+Use one policy per `(game, mapSlug)`, preserving configured variants such as T6 start locations. Do not group ambiguous legacy records by engine map alone. Resolve or exclude ambiguous identities before activation.
+
+| Setting | Proposed initial behavior |
+| --- | --- |
+| Mode | Off, Dry run, Create only, or Create and destroy. Default Off. |
+| Minimum running lobbies | Default 1; a floor for desired ready capacity, subject to available keys and host capacity. |
+| Maximum lobbies | Required explicit limit, at most 20. Count occupied lobbies, registered identities, reservations, and unresolved creations toward admission. |
+| Empty spare target | Default 1. Count empty ready lobbies, not unfilled player slots. |
+| Empty duration before removal | Default 20 minutes; adjustable after the pilot. |
+| Action cooldown | Default 5 minutes between new policy actions, measured from durable admission. Cancellation and safety checks do not wait for cooldown. |
+| Launch settings | Reuse normal map recipes and automatic placement on eligible active hosts. Use current promoted files when provisioning begins. |
+
+Validate integer counts with `1 <= minimum <= maximum <= 20` and `1 <= spare target <= maximum`. Show when saved-key coverage limits the configured maximum; block live activation until enough usable identities have keys. Subsequent key loss blocks affected growth and does not trigger removal. Keys need not occupy consecutive lobby numbers.
+
+Live activation explicitly hands management of the game's map variant to the policy and adopts its eligible existing native lobbies. Show those lobbies, exclusions, current keys, and the first proposed action in the activation preview. Do not activate across unresolved provisioning, collection launches, or ambiguous identity ownership. Dry run observes without acquiring ownership or reserving names, ports, or capacity.
+
+While a policy is live, reject collection launches that would create or remove its lobbies. In exact mode, check the entire removal set, including managed maps absent from the collection. Apply the same check to manual Create, Start, Stop, Rebuild, and Destroy requests that compete with the policy. Explain which policy must be paused, with a link. Existing operator cancellation remains available; cancelling an autoscaling job pauses the policy so it cannot immediately recreate that intent.
+
+Pause the entire policy's decisions while a member is in an active beta, campaign, maintenance operation, host drain, or unresolved recovery. This conservative first-release rule avoids treating temporarily unavailable lobbies as new demand. Those workflows can proceed through their existing gates; autoscaling resumes only after they settle and fresh observations are available. Repeat these checks during Provisioning admission because a conflict can begin after evaluation.
+
+## Counting and decisions
+
+Build a bounded snapshot per map from current identities, evidence, and operations. Deduplicate a registered lobby and its creation job by identity. Classify each member as occupied and ready, empty and ready, creating, removing, stopped, unknown or unhealthy, or protected by other work. Queued and starting lobbies reserve capacity but are not usable spares.
+
+The first release holds new scaling actions whenever an existing member has unknown occupancy, is stopped or unhealthy, or has unresolved lifecycle work. Record the blocker and leave recovery to existing maintenance and operator flows. A confirmed empty map with no registered members can bootstrap from its minimum; it does not need occupancy observations for nonexistent servers.
+
+When the snapshot is complete and eligible, calculate:
+
+```text
+desired running lobbies = min(maximum, max(minimum, occupied ready lobbies + spare target))
+```
+
+This is a bounded target, not a promise that capacity is immediately available. If occupied lobbies consume the maximum, keep them running and report that the spare target cannot be met. Reducing the maximum below the existing count never authorizes removing an occupied lobby.
+
+Evaluate in this order:
+
+1. Reconcile existing autoscaling intent and any unsettled host commands. Cancel an eligible queued removal if players arrive, evidence becomes unknown, the policy changes, a conflict appears, or removal would breach the minimum or spare target.
+2. Apply mode, ownership, evidence, conflict, failure, and cooldown gates. Never submit a second policy action while a prior one remains unresolved. Count external reservations as well as policy jobs against the maximum.
+3. If the ready count is below the desired count, propose one create using the lowest available keyed lobby identity. Recheck identity availability and the policy revision atomically when persisting the request. Capacity waiting must reuse this request rather than add another each tick.
+4. If the ready count exceeds the desired count, select one continuously empty eligible lobby whose removal preserves both the minimum and spare target. Prefer the longest-confirmed empty lobby, with lobby number as a stable tie-breaker. Create only mode records this opportunity without executing it.
+5. Otherwise record a stable no-action reason. Deduplicate unchanged decisions so the 15-second timer does not flood history.
+
+For example, with minimum 1, maximum 4, and one spare, two occupied lobbies plus one empty lobby need no action. If that empty lobby fills, request a fourth. When only one lobby remains occupied and three are empty, the target becomes two; remove surplus empty lobbies one at a time after their empty duration and action cooldown permit it.
+
+Track continuous emptiness against the exact instance generation, using fresh observation timestamps. Re-reading the same sample does not extend the observation history. A positive count, unknown interval, lifecycle transition, or identity replacement resets eligibility. After a controller outage that breaks observation continuity, require a new full empty interval; downtime must not age a lobby into removal.
+
+## Removal and cancellation contract
+
+Automatic removals always use normal safety gates and never set a safety override. Revalidate policy mode and revision, ownership, membership, remaining spare capacity, conflicts, and occupancy immediately before claiming and dispatching removal. Preserve the agent's fresh local occupancy check immediately before mutation.
+
+The existing [cancelRemoval](../src/lib/server/provisioning/history.ts) transaction competes with the worker while a removal is waiting. It cannot undo removal after the worker has claimed it. General cancellation also cannot withdraw an already delivered host command. The agent can hold an occupied command and retry later, so cancelling only the controller row would leave stale destructive intent.
+
+Before enabling Create and destroy, extend the controller/agent contract for autoscaling removals: an occupied or unknown host-local check must invalidate that autoscaling removal attempt rather than retain it for eventual execution. A later attempt requires a new controller evaluation and the full empty interval. Carry policy revision and a bounded authorization expiry, and require renewed controller authorization before an agent retry after delay or restart. Pausing or superseding a policy must invalidate authorization that has not begun mutation. Test these races with the real agent journal behavior; do not assume a controller cancellation implies host acknowledgement.
+
+If mutation has already begun, show that cancellation was too late and let the existing operation reconcile its result. Do not release its identity, key slot, or port until removal and cleanup are confirmed. A cancelled, failed, or ambiguous job must not cause a replacement command to overtake an unsettled delivered command.
+
+Pause on failed or ambiguous automatic operations and link to Provisioning recovery. Capacity waits are normal waiting states, not failures. After explicit recovery, resume from a new snapshot without replaying the old command. This bounds retries and avoids repeatedly creating failing lobbies with the same key or recipe.
+
+## Storage and integration
+
+Add shared policy and decision types plus a server-side evaluator under `src/lib/server/autoscaling/`. Separate a deterministic planner from database reads and Provisioning submission so dry run and live modes evaluate the same rules.
+
+Persist policies with their revision, mode, settings, and timestamps; adopted instance membership; observation continuity; next-action time; and decisions linked to exact Provisioning operation IDs. Store the relevant evidence source and age, counts, intended action, blocker, and policy revision. Do not include keys or full player rosters in decision history.
+
+Use database transactions and uniqueness constraints to enforce one policy per game/map, one unresolved autoscaling intent per policy, and unique lobby allocation across manual launches, collections, and autoscaling. Persist the decision and its operation link together. Process-local in-flight guards prevent overlapping passes, while durable constraints prevent duplicate work after a crash. Add indexes for policy membership, unresolved decisions, and recent history; prune decision history on the existing retention schedule with a proposed 31-day window.
+
+Extend Provisioning source types, admission, history, and backlinks with an autoscaling source containing policy and decision identity. Keep creation recipe selection, placement, operation execution, and cleanup inside the existing services. Add no separate host command executor or per-map RCON polling loop.
+
+Expose authenticated read and update routes using the existing fleet-management authorization rules. Use a revision check for edits and activation so stale forms cannot overwrite newer policy settings. Off and Dry run submit no host mutations. Moving out of a live mode withdraws cancellable policy work, reports unsettled commands, and releases exclusive ownership only when competing mutations can safely proceed.
+
+## Operator experience and measurements
+
+Add an Autoscaling view linked from Fleet, Collections, and Metrics. Show each map's mode, occupied and empty ready counts, pending work, minimum, maximum, spare target, key coverage, last evaluation, next eligible action, and current blocker. Detail includes the policy editor, affected lobbies, recent decisions, Provisioning links, and pause/resume controls. Dry run actions must be visibly hypothetical.
+
+Activation and edits preview the resulting target and any proposed removals. Pause stops new decisions immediately and shows which queued actions were withdrawn and which delivered operations remain unresolved. Do not claim the fleet was restored when a policy is disabled.
+
+Record empty-spare deficit minutes, unknown-status minutes, empty lobby-hours, action frequency, capacity/key blocks, cancellations, failures, and request-to-ready durations. Use Provisioning milestones for queue time and readiness, but measure browser visibility and actual successful joins separately in the pilot: a healthy container or RCON response alone does not establish player joinability.
+
+Compare equivalent weekly periods and record game/runtime versions, player-hours, host CPU and memory, and provisioning latency. “Time with players” will naturally rise when idle lobbies leave its denominator; it does not by itself prove increased demand or CPU savings. Dry run estimates are conditional on the observed fleet, not a simulation of how players would redistribute after removal. Use measured creation and join latency to tune spare targets before broad rollout.
+
+## Delivery sequence and acceptance
+
+Complete and validate each phase in a separate commit before starting the next.
+
+1. **Policy model and planner.** Add additive schema changes, identity-aware snapshots, policy validation, decision types, and the pure evaluator. Keep all modes non-mutating until admission integration exists. Acceptance: deterministic fixtures cover the target formula, variant separation, incomplete evidence, saved-key gaps, bounds, and continuous empty timing.
+2. **Dry run and visibility.** Register the requester, persist bounded decision history, and add the policy UI/API with dry run enabled. Acceptance: repeated ticks and controller restarts produce understandable decisions without creating Provisioning jobs or reserving identities, ports, or hosts. Begin the two-week observation period on representative maps.
+3. **Ownership and live creation.** Add transactional intent, Provisioning source eligibility for Create/Destroy, collection/manual conflict checks, and Create only mode. Acceptance: no duplicate creation under concurrent ticks or submissions; no growth beyond the maximum; no capacity bypass; blocked creates remain single durable jobs. Pilot one or two maps after reviewing dry run evidence.
+4. **Safe removal.** Implement continuous-empty selection, admission rechecks, queued cancellation, and the agent authorization contract above. Acceptance: positive/unknown occupancy blocks destruction; a held command cannot remove a lobby later without renewed eligibility; policy pause, revision changes, and new protected work invalidate pending intent. Enable Create and destroy only after these checks pass.
+5. **Pilot review and rollout.** Compare CPU and empty lobby-hours alongside spare availability, successful joins, failures, and creation latency. Expand one map at a time only if savings are measurable, ordinary gameplay is preserved, and loss of spare availability stays within the pilot's agreed tolerance. Document the tuned settings and operating procedure.
+
+Required behavioral scenarios across these phases:
+
+| Scenario | Expected result |
+| --- | --- |
+| Last spare fills, including a high-round partially filled session | One create when eligible; free player slots do not count as spares. |
+| Create is queued, starting, or awaiting verification | Reservation counts once; it is not advertised as ready and no duplicate is queued. |
+| Every eligible host is full or unavailable | Existing job waits; other policies and worker tasks continue. |
+| Key is missing, removed, or rejected | Clear blocker or paused failure; no alternate identity receives the same key automatically. |
+| Occupancy evidence conflicts, expires, or arrives from an old lifecycle | Positive current occupancy wins; invalid evidence cannot establish emptiness. |
+| Players join before claim, after dispatch, or while the agent holds removal | Queued work is withdrawn where possible; agent safety prevents occupied destruction; held autoscaling intent cannot execute later without a fresh decision. |
+| Process restarts before or after intent submission | Existing operation is recovered by durable identity without replay or duplicate admission. |
+| Collection exact match omits a managed map | Preview and submission report the ownership conflict, including stale-preview races. |
+| Beta, campaign, maintenance, host drain, or recovery begins after evaluation | Admission rechecks block scaling and existing work retains its lifecycle ownership. |
+| Operator pauses, changes limits, or cancels a policy job | Unstarted intent is invalidated; unsettled commands stay visible and keep reservations until reconciled. |
+| A name is reused after confirmed deletion | Old observations, cooldown decisions, and commands cannot apply to the replacement generation. |
+
+Run targeted Vitest coverage for planner, collections, provisioning, cancellation, telemetry, and routes, plus Python agent tests for the removal contract. Before each implementation phase is committed, run the relevant checks; before live release, run `npm run check`, `npm test`, `npm run test:agent`, and `npm run build`. Verify the UI with keyboard navigation and narrow layouts, including stale evidence, blocked capacity, and partial cancellation states.
+
+## Rollback
+
+Provide a global pause as well as per-policy pause. It stops new decisions and withdraws unstarted work using the same cancellation contract. Keep the controller and agent running to settle delivered operations and cleanup; do not delete policies or reservations to bypass recovery.
+
+After outstanding operations settle, release map ownership. Operators can restore a fixed lineup through Collections using normal capacity and empty-server gates. Turning autoscaling off does not recreate removed lobbies or remove lobbies it created. Retain decision and Provisioning history for review, and keep the additive schema compatible with all policies Off.
+
+## Implementation and validation record
+
+The implementation adds the pure evaluator and persistent policy model, dry-run requester and UI, exclusive map ownership with capacity-aware creation, and revocable removal authorization. It creates no production policies. The authorization claim is the final cancellation boundary; agents invalidate an occupied or expired attempt and require a new decision rather than retrying it later. The local claim window is five seconds, measured from the start of the authorization request, with another occupancy check before mutation.
+
+Validation includes planner and snapshot tests, duplicate-admission and stale-preview tests, collection exact-match conflicts, operator cancellation, authorization expiry and host identity, controller/agent occupancy races, and agent restart recovery. Agent packaging publishes version 1.7.0 with the new capability.
+
+The full controller suite reported 1,934 passing tests and two existing failures in `server-detail-api.test.ts` (plugin classification) and `watchdog.test.ts` (an occupied-telemetry RCON expectation). Both failures reproduce independently at the pre-implementation commit `a22461a`. They are not autoscaling regressions. Targeted controller tests, all 228 Python agent tests, type checking, and the production build passed.
+
+An isolated Firefox review exercised policy creation, preview, save, keyboard focus, and 1440, 390, and 320 pixel layouts. A narrow-layout heading issue found during review was corrected. The preview used a temporary database and no production host credentials. Production CPU savings, external joinability, spare targets, and acceptable deficit durations still require the two-week observation period and operator-run pilot.

src/lib/autoscaling.ts

@@ -0,0 +1,53 @@
+import { zombieMapBySlug, type ZombieGame } from './map-aliases.js';
+
+export const autoscalingModes = ['off', 'dry-run', 'create-only', 'create-destroy'] as const;
+export type AutoscalingMode = typeof autoscalingModes[number];
+export interface ScalingSettings {
+    game: ZombieGame; mapSlug: string; mode: AutoscalingMode;
+    minimum: number; maximum: number; spares: number; emptyMinutes: number; cooldownMinutes: number;
+}
+export interface ScalingPolicy extends ScalingSettings {
+    id: string; revision: number; nextActionAt: number; pausedReason: string | null; updatedAt: string;
+}
+export interface ScalingMember {
+    name: string; identity: string; state: 'occupied' | 'empty' | 'creating' | 'removing' | 'stopped' | 'unknown' | 'protected';
+    players: number | null; observedAt: number | null; source: string | null;
+    emptySince: number | null; blocker: string | null; operationId: string | null;
+    emptyStartedAt?: number | null;
+}
+export interface ScalingSnapshot {
+    members: ScalingMember[]; reservedNames: string[]; keyedNames: string[]; blockers: string[];
+}
+export interface ScalingDecision {
+    action: 'create' | 'destroy' | 'wait'; reason: string; server: string | null;
+    desired: number; occupied: number; empty: number; pending: number; unknown: number;
+}
+export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] }
+export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
+export interface ScalingView extends ScalingPreview {
+    history: ScalingHistory[]; lastEvaluatedAt: number | null;
+    totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
+    operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null };
+    ownsMap: boolean;
+    unsettled: { id: string; action: string; state: string; stage: string }[];
+}
+export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
+export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy';
+
+export function scalingSettings(input: unknown): ScalingSettings {
+    if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.');
+    const value = input as Record<string, unknown>;
+    const { game, mapSlug } = value;
+    if ((game !== 't5' && game !== 't6') || typeof mapSlug !== 'string' || zombieMapBySlug(game, mapSlug)?.slug !== mapSlug) throw new Error('Choose a supported game and map variant.');
+    const mode = value.mode ?? 'off';
+    if (!autoscalingModes.includes(mode as AutoscalingMode)) throw new Error('Choose a valid autoscaling mode.');
+    const integer = (key: string, fallback: number | undefined, max: number) => {
+        const n = value[key] ?? fallback;
+        if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > max) throw new Error(`${key} must be an integer from 1 to ${max}.`);
+        return n;
+    };
+    const minimum = integer('minimum', 1, 20), maximum = integer('maximum', undefined, 20), spares = integer('spares', 1, 20);
+    if (minimum > maximum || spares > maximum) throw new Error('Minimum and spare target cannot exceed the maximum.');
+    return { game, mapSlug, mode: mode as AutoscalingMode, minimum, maximum, spares,
+        emptyMinutes: integer('emptyMinutes', 20, 1440), cooldownMinutes: integer('cooldownMinutes', 5, 1440) };
+}

src/lib/navigation.ts

@@ -7,6 +7,7 @@ export const navigationGroups: NavigationGroup[] = [
     { id: 'fleet', label: 'Fleet', items: [
         { label: 'Overview', href: '/', shortcut: 'f', matches: path => path === '/' || path.startsWith('/servers/') },
         { label: 'Collections', href: '/collections' },
+        { label: 'Autoscaling', href: '/autoscaling' },
         { label: 'Hosts', href: '/hosts', shortcut: 'o', matches: path => path === '/hosts' || /^\/hosts\/(?!operations(?:\/|$))/.test(path) },
         { label: 'Capacity', href: '/capacity' }
     ] },

src/lib/provisioning-detail.ts

@@ -77,7 +77,7 @@ export interface ProvisioningDeployment {
     expected_map?: string | null;
 }
 export interface ProvisioningSource {
-    kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog';
+    kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog' | 'autoscaling';
     source_id: string;
     server?: string;
     label: string;

src/lib/server/autoscaling/admission.ts

@@ -0,0 +1,33 @@
+import { db } from '../db/index.js';
+import { planScaling } from './planner.js';
+import { scalingSnapshot } from './snapshot.js';
+import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
+import { deploymentStage } from '../provisioning/history.js';
+
+export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
+export function scalingIntent(operationId: string): ScalingIntent | undefined {
+    return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
+}
+/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
+export function scalingAdmission(operationId: string): string | null {
+    const intent = scalingIntent(operationId);
+    if (!intent) return null;
+    const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
+    let reason: string | null = null;
+    const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
+    if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
+    else {
+        const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' };
+        const requested = JSON.parse(intent.decision) as ScalingDecision;
+        if (requested.action === 'destroy' && row.mode !== 'create-destroy') return 'Automatic removal is disabled.';
+        const snapshot = scalingSnapshot(policy, Date.now(), operationId);
+        if (requested.action === 'create') {
+            snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
+            snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
+        }
+        const current = planScaling(policy, snapshot, Date.now());
+        if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
+    }
+    if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
+    return reason;
+}

src/lib/server/autoscaling/authorization.ts

@@ -0,0 +1,52 @@
+import { db } from '../db/index.js';
+import { scalingAdmission, scalingIntent } from './admission.js';
+import { getHost } from '../hosts/index.js';
+
+export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
+/** Authorization is attached to the immutable host command inside its submission transaction. */
+export function removalAuthorization(operationId: string, hostId: string): object | null {
+    const intent = scalingIntent(operationId);
+    if (!intent) return null;
+    if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
+    const reason = scalingAdmission(operationId);
+    if (reason) throw new Error(reason);
+    const expiresAt = Date.now() + 60_000;
+    db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
+        .run(operationId, intent.policy_id, intent.revision, expiresAt);
+    return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
+}
+
+/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
+export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
+    return db().transaction(() => {
+        const now = Date.now();
+        const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
+            JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
+            JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+            JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
+            WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
+            { operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
+        if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
+            || row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
+            || row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
+        const reason = scalingAdmission(row.operation_id);
+        if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
+        db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
+        // Only a short final occupancy check may occur after this boundary.
+        return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
+    })();
+}
+
+export function revokePolicyRemovals(policyId: string): void {
+    if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
+    db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
+    const at = new Date().toISOString();
+    const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
+        JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
+        WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
+    for (const command of commands) {
+        const result = JSON.stringify({ autoscalingInvalidated: true });
+        db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
+        db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
+    }
+}

src/lib/server/autoscaling/index.ts

@@ -0,0 +1,199 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { db } from '../db/index.js';
+import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
+import { emptyObservation, planScaling } from './planner.js';
+import { scalingSnapshot } from './snapshot.js';
+import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
+import { cancelCreation } from '../provisioning/cancel-creation.js';
+import { scalingAdmission } from './admission.js';
+import { destroyServer } from '../provisioning/instances.js';
+import { cancelRemoval } from '../provisioning/history.js';
+import { autoscalingRemovalCapability, revokePolicyRemovals } from './authorization.js';
+import { getHost } from '../hosts/index.js';
+import { getServer } from '../fleet/servers.js';
+
+export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
+interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
+const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at });
+export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
+export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
+export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
+
+export function previewScaling(input: unknown, id?: string, revision?: number): ScalingPreview {
+    const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
+    if (id && !existing) throw new ScalingError('Policy not found.', 404);
+    if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
+    if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
+    const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0,
+        nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
+    const snapshot = scalingSnapshot(policy);
+    const blockers: string[] = [];
+    if (liveScaling(policy.mode)) {
+        if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`);
+        blockers.push(...snapshot.blockers);
+        for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
+        if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
+            const hostId = getServer(member.name)?.host_id;
+            if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
+        }
+        if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
+        try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
+    }
+    const decision = planScaling(policy, snapshot, Date.now());
+    const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(),
+        members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
+    return { policy, snapshot, decision, token, blockers };
+}
+
+export function saveScaling(input: unknown, id?: string, revision?: number, token?: string): ScalingPolicy {
+    return db().transaction(() => {
+        const preview = previewScaling(input, id, revision);
+        if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409);
+        if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
+        const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
+        if (id) revokePolicyRemovals(id);
+        if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
+        db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
+            ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
+        // An edit starts a new observation interval; earlier inactivity is not authorization.
+        db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
+        db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
+        if (liveScaling(policy.mode)) for (const member of preview.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
+            .run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
+        reconcileScaling(getScalingPolicy(nextId)!);
+        return getScalingPolicy(nextId)!;
+    })();
+}
+
+export function setScalingGlobalPause(paused: boolean): void {
+    db().transaction(() => {
+        db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
+        if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
+    })();
+}
+
+function reconcileScaling(policy: ScalingPolicy): void {
+    const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
+        FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
+        { id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
+    for (const intent of intents) {
+        if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
+            && (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
+        if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
+            revokePolicyRemovals(policy.id);
+            if (!intent.command_id) cancelRemoval(intent.operation_id, false);
+        }
+        const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
+        const terminal = ['done','warning','failed','cancelled'].includes(current.state);
+        const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
+        const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
+        const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
+            && !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
+        const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
+        if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
+            db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
+        }
+        if (terminal && settled) {
+            db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
+            if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
+        }
+    }
+    if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
+        db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
+    }
+}
+
+export async function runAutoscaling(): Promise<boolean> {
+    const now = Date.now();
+    let submitted = false;
+    for (const initial of listScalingPolicies()) {
+        reconcileScaling(initial);
+        const policy = getScalingPolicy(initial.id)!;
+        if (policy.mode === 'off') continue;
+        db().transaction(() => {
+            const snapshot = scalingSnapshot(policy, now);
+            if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
+            const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
+            if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
+            const decision = planScaling(policy, snapshot, now);
+            for (const member of snapshot.members) {
+                // Keep the observation interval that justified a pending removal;
+                // admission reads fresh evidence and revokes it on any new blocker.
+                if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
+                const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
+                db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
+                    ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
+            }
+            // Evidence includes provenance and timestamps but never keys or player rosters.
+            const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
+            const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
+            const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
+            const execute = liveScaling(policy.mode) && (decision.action === 'create' || policy.mode === 'create-destroy' && decision.action === 'destroy');
+            const decisionId = randomUUID();
+            if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
+            else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
+                .run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
+            if (execute) {
+                db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
+                try {
+                    db().transaction(() => {
+                        const result = decision.action === 'create'
+                            ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
+                                selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
+                            : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
+                        if (!result.ok) throw new Error(result.message);
+                        if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
+                        db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
+                        db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
+                        db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
+                        submitted = true;
+                    })();
+                } catch (error) {
+                    db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
+                    db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
+                }
+            }
+            db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
+                .run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
+        })();
+    }
+    return submitted;
+}
+
+export function pruneAutoscaling(now = Date.now()): void {
+    const cutoff = now - 31 * 86400_000;
+    db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
+    db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
+    db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
+}
+
+export function scalingReport(): ScalingReport {
+    const policies: ScalingView[] = listScalingPolicies().map(policy => {
+        const snapshot = scalingSnapshot(policy);
+        if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
+        const decision = planScaling(policy, snapshot, Date.now());
+        const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
+            FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
+            WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
+            { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
+            .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
+        const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
+        const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
+        // Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
+        const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
+            FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
+            COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
+            COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
+            COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
+        const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
+            COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
+            COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
+            AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
+            MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
+            FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
+            WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
+        return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
+    });
+    return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
+}

src/lib/server/autoscaling/ownership.ts

@@ -0,0 +1,25 @@
+import { db } from '../db/index.js';
+import { getServer } from '../fleet/servers.js';
+import { mapIdentity } from './snapshot.js';
+
+export function scalingOwner(game: string, mapSlug: string): string | null {
+    return (db().prepare('SELECT id FROM autoscaling_policies WHERE game=? AND map_slug=? AND owns_map=1').get(game, mapSlug) as { id: string } | undefined)?.id ?? null;
+}
+export function scalingOwnershipBlocker(server: string): string | null {
+    const identity = mapIdentity(getServer(server) ?? { name: server });
+    const owner = identity && scalingOwner(identity.game, identity.mapSlug);
+    return owner ? `Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}` : null;
+}
+export function assertScalingOwnership(game: string, mapSlug: string, decisionId?: string): void {
+    const owner = scalingOwner(game, mapSlug);
+    if (!owner) { if (decisionId) throw new Error('Autoscaling no longer owns this map.'); return; }
+    if (decisionId && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
+    throw new Error(`Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}`);
+}
+/** Operator cancellation is a policy pause, not a fresh scaling signal. */
+export function pauseScalingForOperation(operationId: string): void {
+    const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
+    if (!row) return;
+    db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
+    db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
+}

src/lib/server/autoscaling/planner.ts

@@ -0,0 +1,43 @@
+import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+
+export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
+/** Only distinct, continuous evidence can advance an empty interval. */
+export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
+    const stamp = member.observedAt;
+    const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
+    const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
+        && (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
+        && previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
+        && now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
+    return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
+}
+
+export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
+    const { members } = snapshot;
+    const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
+    const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
+    const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
+    const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
+    const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
+        ({ action, reason, server, desired, occupied, empty, pending, unknown });
+    if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
+    if (policy.pausedReason) return result('wait', policy.pausedReason);
+    if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
+    const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
+    if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
+    if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
+    if (policy.nextActionAt > now) return result('wait', 'Action cooldown is active.');
+    if (occupied + empty < desired) {
+        if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
+        const available = snapshot.keyedNames.filter(name => !snapshot.reservedNames.includes(name)).sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
+        return available.length ? result('create', 'Replenish empty lobby capacity.', available[0]) : result('wait', 'No unused lobby identity has a saved key.');
+    }
+    if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
+        const candidates = members.filter(m => m.state === 'empty' && m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000)
+            .sort((a, b) => a.emptySince! - b.emptySince! || a.name.localeCompare(b.name, 'en', { numeric: true }));
+        if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
+        return result('wait', 'Waiting for a continuous empty interval.');
+    }
+    return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
+}

src/lib/server/autoscaling/schema.ts

@@ -0,0 +1,37 @@
+import type Database from 'better-sqlite3';
+
+export function ensureAutoscalingSchema(handle: Database.Database): void {
+    handle.exec(`
+        CREATE TABLE IF NOT EXISTS autoscaling_policies (
+            id TEXT PRIMARY KEY, game TEXT NOT NULL, map_slug TEXT NOT NULL,
+            revision INTEGER NOT NULL DEFAULT 1, settings TEXT NOT NULL,
+            mode TEXT NOT NULL DEFAULT 'off', next_action_at INTEGER NOT NULL DEFAULT 0,
+            paused_reason TEXT, owns_map INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL,
+            UNIQUE(game,map_slug)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_members (
+            policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), server TEXT NOT NULL,
+            identity TEXT NOT NULL, observation TEXT NOT NULL, adopted INTEGER NOT NULL DEFAULT 0,
+            PRIMARY KEY(policy_id,server)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_decisions (
+            id TEXT PRIMARY KEY, policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id),
+            revision INTEGER NOT NULL, evaluated_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL,
+            fingerprint TEXT NOT NULL, decision TEXT NOT NULL, evidence TEXT NOT NULL,
+            operation_id TEXT UNIQUE REFERENCES server_deployments(id), resolved INTEGER NOT NULL DEFAULT 1
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS autoscaling_one_intent ON autoscaling_decisions(policy_id) WHERE resolved=0;
+        CREATE INDEX IF NOT EXISTS autoscaling_history ON autoscaling_decisions(policy_id,evaluated_at DESC);
+        CREATE INDEX IF NOT EXISTS autoscaling_decision_retention ON autoscaling_decisions(last_seen_at) WHERE resolved=1;
+        CREATE TABLE IF NOT EXISTS autoscaling_samples (
+            policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), sampled_at INTEGER NOT NULL,
+            mode TEXT NOT NULL, occupied INTEGER NOT NULL, empty INTEGER NOT NULL, unknown INTEGER NOT NULL,
+            deficit INTEGER NOT NULL, PRIMARY KEY(policy_id,sampled_at)
+        );
+        CREATE TABLE IF NOT EXISTS autoscaling_authorizations (
+            operation_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL,
+            expires_at INTEGER NOT NULL, claimed_at INTEGER, revoked_at INTEGER
+        );
+        CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at);
+    `);
+}

src/lib/server/autoscaling/snapshot.ts

@@ -0,0 +1,77 @@
+import { db } from '../db/index.js';
+import { listServers, type ServerRow } from '../fleet/servers.js';
+import { observationReader } from '../fleet/status.js';
+import { parseServerName, zombieMaps } from '../../map-aliases.js';
+import { freshStatusTimestamp } from '../../server-freshness.js';
+import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
+import { emptyObservation, type EmptyObservation } from './planner.js';
+
+export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
+    const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
+    if (!game) return null;
+    if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
+    if (server.map_slug) return null;
+    if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
+    const matches = zombieMaps(game).filter(m => m.canonical === server.map);
+    return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
+}
+
+export function protectedWork(server: string, exceptOperation = ''): string | null {
+    const handle = db();
+    if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
+    if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
+    if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
+    if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
+    return null;
+}
+
+export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
+    const handle = db(), observe = observationReader();
+    const rows = listServers();
+    const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
+    const blockers: string[] = [];
+    for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
+    const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
+    const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
+        JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
+    const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
+        const observation = observe(row), operation = operations.find(o => o.server === row.name);
+        const protection = protectedWork(row.name, exceptOperation);
+        const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
+        const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
+        let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
+        let players = observation.health === 'online' ? observation.last_players : null;
+        let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
+        let source = observation.status_source;
+        // Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
+        if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
+            && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
+            players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
+        }
+        if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
+        else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
+        else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
+        else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
+        const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
+        return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
+            emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
+    });
+    for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
+        const operation = operations.find(o => o.server === row.name);
+        members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
+    }
+    const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
+    for (const member of members) {
+        const saved = retained.find(r => r.server === member.name);
+        const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
+        member.emptySince = emptyObservation(member, previous, now).emptySince;
+    }
+    const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
+    for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
+        reservedNames.add(row.name);
+        if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
+    }
+    const keyedNames = (handle.prepare("SELECT server_id AS name FROM server_keys WHERE length(trim(pluto_key))>0").all() as { name: string }[]).filter(matches).map(r => r.name);
+    return { members, reservedNames: [...reservedNames], keyedNames, blockers };
+}

src/lib/server/db/index.ts

@@ -10,6 +10,7 @@ import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
 import { initializeFleetSettings } from './fleet-settings.js';
 import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
+import { ensureAutoscalingSchema } from '../autoscaling/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateFileBlobs } from './file-blobs.js';
 import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
@@ -275,7 +276,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void {
     for (const name of ['provisioning_sources', 'deployment_servers']) {
         const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
         const replacement = name === 'provisioning_sources'
-            ? (sql.includes("'watchdog'") ? sql : sql.replace("'campaign','beta','maintenance'", "'campaign','beta','maintenance','watchdog'"))
+            ? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'"))
             : sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
         if (sql === replacement) continue;
         const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
@@ -395,6 +396,7 @@ export function db(): Database.Database {
         backfillLeaderboard(handle);
         initializeFleetSettings(handle);
         migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
+        migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
         // File-blob migration removes the legacy content column altogether.
         // Reordering it first would unnecessarily copy every artifact twice.
         migrateFileBlobs(handle);

src/lib/server/db/schema.sql

@@ -527,7 +527,7 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
 -- history retention, while current pointers live on the caller's server rows.
 CREATE TABLE IF NOT EXISTS provisioning_sources (
     operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
-    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog')),
+    kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog','autoscaling')),
     source_id TEXT NOT NULL,
     server TEXT NOT NULL,
     PRIMARY KEY (operation_id, kind, source_id, server)

src/lib/server/fleet/collections.ts

@@ -1,3 +1,4 @@
+import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
 import { fleetSettings } from '../settings.js';
 import { randomUUID } from 'node:crypto';
 import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
@@ -97,6 +98,7 @@ function allocate(entries: BlueprintEntry[], counts = coverage(entries)): Collec
 
 function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
     const blockers: string[] = [];
+    for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); }
     if (!servers.length) return blockers;
     if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
     if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
@@ -113,6 +115,8 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
     };
     if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
     for (const server of removals) {
+        const owned = scalingOwnershipBlocker(server.name);
+        if (owned) blockers.push(owned);
         const blocker = activeBetaRemovalBlocker(server.name);
         if (blocker) blockers.push(blocker);
     }

src/lib/server/pipeline/worker.ts

@@ -1,4 +1,5 @@
 import { pruneHistory, pruneSampleHistory } from '../retention.js';
+import { runAutoscaling, pruneAutoscaling } from '../autoscaling/index.js';
 import { config } from '../config.js';
 import { processProvisioning } from '../provisioning/worker.js';
 import { sweep } from './sweep.js';
@@ -54,8 +55,9 @@ function reconcileRequesters(): void {
         ['campaign', 'Reconciling campaigns', sweep],
         ['beta', 'Reconciling beta tests', betaTick],
         ['maintenance', 'Checking maintenance and recovery', runMaintenance],
+        ['autoscaling', 'Evaluating lobby autoscaling', runAutoscaling],
         ['game-files', 'Removing unused game files', cleanupControllerLibraries],
-        ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); }]
+        ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); }]
     ];
     for (const [key, label, work] of requesters) {
         if (requesterFlights.has(key)) continue;
@@ -65,7 +67,7 @@ function reconcileRequesters(): void {
         }
         const flight = trackActivity(key, label, work)
             .catch(err => { console.error(`[worker] ${key} error:`, err); })
-            .then(async () => { if (key !== 'game-files' && key !== 'retention') await tick(true); })
+            .then(async result => { if (key !== 'game-files' && key !== 'retention' && (key !== 'autoscaling' || result === true)) await tick(true); })
             .finally(() => { requesterFlights.delete(key); });
         requesterFlights.set(key, flight);
     }

src/lib/server/provisioning/cancel-creation.ts

@@ -1,3 +1,4 @@
+import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
 import { nativeInstance } from '../backends/native.js';
 import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
@@ -21,11 +22,12 @@ export function canCancelCreation(id: string): boolean {
 }
 
 /** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
-export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
+export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = true): string | null {
     return db().transaction(() => {
         const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
         if (prior) return prior.cleanup_id;
         if (!canCancelCreation(id)) return null;
+        if (pausePolicy) pauseScalingForOperation(id);
         const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
             LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
             { server: string; operation_id: string | null; stage_operation_id: string | null };

src/lib/server/provisioning/cancellation.ts

@@ -1,3 +1,4 @@
+import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from '../hosts/schema.js';
 import { recordProgress } from './progress.js';
@@ -21,6 +22,7 @@ export function cancelProvisioningJob(id: string, reason = 'Cancelled by operato
     return db().transaction(() => {
         if (provisioningCancelled(id)) return true;
         if (!canCancelProvisioningJob(id)) return false;
+        pauseScalingForOperation(id);
         const at = new Date().toISOString();
         // Only commands never delivered can be withdrawn. Delivered commands retain
         // their immutable IDs and receipts; replacements wait for their resolution.

src/lib/server/provisioning/history.ts

@@ -1,3 +1,4 @@
+import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
 import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
@@ -59,6 +60,7 @@ function warningMessages(raw: string): string[] {
 
 function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
     const id = source.source_id;
+    if (source.kind === 'autoscaling') return { ...source, label: 'Autoscaling policy', href: `/autoscaling#${encodeURIComponent(id)}` };
     if (source.kind === 'campaign') {
         const row = db().prepare(`SELECT d.action, d.beta_id, s.name FROM deployments d
             LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1))
@@ -146,12 +148,13 @@ export function canCancelRemoval(id: string): boolean {
 }
 
 /** Competes atomically with the worker's claim, before any external removal. */
-export function cancelRemoval(id: string): boolean {
+export function cancelRemoval(id: string, pausePolicy = true): boolean {
     return db().transaction(() => {
         const finishedAt = new Date().toISOString();
         const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
             WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
         if (!result.changes) return false;
+        if (pausePolicy) pauseScalingForOperation(id);
         recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
             message: 'Queued removal cancelled. The server and its files were kept.',
             details: { state: 'cancelled', finishedAt } } });
@@ -306,7 +309,7 @@ function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, activ
     const cache = new Map<string, ProvisioningSource>();
     for (const row of rows) {
         const sources = byOperation.get(row.id) ?? [];
-        const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog):(.+)$/);
+        const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/);
         if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
         row.sources = sources.map(source => {
             const key = `${source.kind}:${source.source_id}`;

src/lib/server/provisioning/instances.ts

@@ -1,3 +1,5 @@
+import { scalingOwnershipBlocker, assertScalingOwnership } from '../autoscaling/ownership.js';
+import { mapIdentity } from '../autoscaling/snapshot.js';
 import { db } from '../db/index.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import type { GameMode, ZombieGame } from '../fleet/map-aliases.js';
@@ -67,7 +69,13 @@ function hasActiveCreation(name: string): boolean {
     return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
 }
 
-export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
+export function destroyServer(name: string, triggeredBy = 'ui', autoscalingDecisionId?: string): DestroyServerResult & { deploymentId?: string } {
+    const owned = scalingOwnershipBlocker(name);
+    if (autoscalingDecisionId) {
+        const identity = mapIdentity(getServer(name) ?? { name });
+        if (!identity) return { ok: false, kind: 'conflict', message: 'Map identity unavailable.', warnings: [] };
+        assertScalingOwnership(identity.game, identity.mapSlug, autoscalingDecisionId);
+    } else if (owned) return { ok: false, kind: 'conflict', message: owned, warnings: [] };
     const cancelled = cancelQueuedNativeCreation(name, triggeredBy);
     if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled };
     const blocker = activeBetaRemovalBlocker(name);

src/lib/server/provisioning/native.ts

@@ -1,3 +1,6 @@
+import { assertScalingOwnership } from '../autoscaling/ownership.js';
+import { removalAuthorization } from '../autoscaling/authorization.js';
+import { scalingIntent } from '../autoscaling/admission.js';
 import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
 import { concurrencyBlocker, preparingJobs } from './concurrency.js';
 import { isIP } from 'node:net';
@@ -17,7 +20,7 @@ import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch
 import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
-import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
+import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, cancelRemoval } from './history.js';
 import { recordProgress, type CheckInput } from './progress.js';
 import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
@@ -152,11 +155,14 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
 
 /** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */
 export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection;
-    plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan }): CreateServerResult {
+    plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan; autoscalingDecisionId?: string }): CreateServerResult {
     try {
         return db().transaction(() => {
+            assertScalingOwnership(input.game, input.mapSlug, input.autoscalingDecisionId);
             const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`;
-            if (input.server !== expected || nativeInstance(input.server)) throw new Error('Available lobby changed. Review a fresh preview.');
+            const reserved = db().prepare(`SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=?
+                AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))`).get(input.server);
+            if ((!input.autoscalingDecisionId && input.server !== expected) || nativeInstance(input.server) || getServer(input.server) || reserved) throw new Error('Available lobby changed. Review a fresh preview.');
             const key = input.plutoKey?.trim() || getServerKey(input.server);
             if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`);
             saveServerKey(input.server, key);
@@ -402,7 +408,7 @@ function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boo
 }
 
 /** Capacity history and host ranking run in the background reader, one job at a time. */
-async function placeQueuedCreation(job: Job, instance: NativeInstance | null): Promise<NativeInstance | null> {
+async function placeQueuedCreation(job: Job, instance: NativeInstance | null, eligibility: (job: Job) => string | null): Promise<NativeInstance | null> {
     const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
     const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!;
     const network = prior.automatic ? prior.network ?? {} : prior;
@@ -413,6 +419,7 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
     return db().transaction(() => {
         const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
         if (state?.state !== 'queued') return null;
+        if (eligibility(job)) return null;
         const live = nativeInstance(job.server);
         if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null;
         const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
@@ -467,7 +474,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         const capacityBlocker = concurrencyBlocker(job.id, job.action);
         if (capacityBlocker) { hold(job, capacityBlocker); return; }
         if (eligibility(job)) return;
-        instance = await placeQueuedCreation(job, instance);
+        instance = await placeQueuedCreation(job, instance, eligibility);
         if (!instance) return;
         job.instance_id = instance.id; job.generation = instance.generation;
     }
@@ -493,6 +500,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
+            if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) {
+                db().transaction(() => {
+                    db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?")
+                        .run(new Date().toISOString(), job.id);
+                    recordProgress(job.id, { blocker: null, event: { kind: 'outcome', message: 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', details: { state: 'cancelled' } } });
+                })();
+                return;
+            }
             if (migration) {
                 if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
                     db().transaction(() => {
@@ -654,7 +669,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
                 WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
             if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; }
-            if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return false; }
+            if (occupancy.players !== 0) {
+                if (job.action === 'destroy' && scalingIntent(job.id)) cancelRemoval(job.id, false);
+                else hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.');
+                return false;
+            }
         }
         return true;
     };
@@ -724,6 +743,13 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 ? 'Host eligibility checked; player safety checks bypassed by admin override' : 'Host eligibility and safety checks passed' } });
             const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
             if (overrideSafety) payload.safetyOverride = true;
+            if (job.action === 'destroy') {
+                const authorization = removalAuthorization(job.id, instance!.host_id);
+                if (authorization) {
+                    if (overrideSafety) throw new Error('Autoscaling never bypasses player safety.');
+                    payload.autoscaling = authorization;
+                }
+            }
             if (prepared) {
                 deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
                     checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });

src/lib/server/provisioning/requests.ts

@@ -1,3 +1,4 @@
+import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { db } from '../db/index.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -120,6 +121,8 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
 
 export function requestLifecycle(server: string, action: RuntimeAction) {
     if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
+    const owned = action !== 'restart' ? scalingOwnershipBlocker(server) : null;
+    if (owned) return { ok: false as const, status: 409, error: owned };
     if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
     return db().transaction(() => {
         const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;

src/lib/server/provisioning/safety-override.ts

@@ -19,6 +19,7 @@ export function safetyOverrideAvailability(id: string): { applicable: boolean; u
         AND d.state IN ('queued','waiting') AND d.submitted_at IS NULL AND n.operation_id IS NULL
         AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id)
         AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.cleanup_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM autoscaling_decisions a WHERE a.operation_id=d.id)
         AND NOT EXISTS (SELECT 1 FROM provisioning_safety_overrides o WHERE o.deployment_id=d.id)`)
         .get(id) as { host_id: string } | undefined;
     if (!target) return { applicable: false, unavailableReason: null };

src/lib/server/provisioning/worker.ts

@@ -1,3 +1,4 @@
+import { scalingAdmission } from '../autoscaling/admission.js';
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { db } from '../db/index.js';
 import { getServer } from '../fleet/servers.js';
@@ -93,6 +94,8 @@ export const processLifecycle = processProvisioning;
 
 function nativePass(options: { wait?: boolean }): Promise<void> {
     return processNativeProvisioning(operation => {
+        const scaling = scalingAdmission(operation.id);
+        if (scaling) return scaling;
         if (operation.action === 'create') return null;
         if (operation.action === 'destroy') {
             const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL

src/lib/ui/provisioning-presentation.ts

@@ -179,7 +179,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
     if (!record(value.activity) || !number(value.activity.total) || !(value.activity.latestId === null || number(value.activity.latestId))
         || !(value.activity.failureExcerpt === null || event(value.activity.failureExcerpt))) return false;
     if (!Array.isArray(value.sources) || !value.sources.every(source => record(source)
-        && ['campaign', 'beta', 'maintenance', 'watchdog'].includes(source.kind as string)
+        && ['campaign', 'beta', 'maintenance', 'watchdog', 'autoscaling'].includes(source.kind as string)
         && text(source.source_id) && text(source.label) && localLink(source.href))) return false;
     if (!(value.collection === null || record(value.collection) && text(value.collection.id) && text(value.collection.name))) return false;
     if (!record(value.capacity) || !timestamp(value.capacity.evaluatedAt) || !Array.isArray(value.capacity.pools)

src/routes/+page.svelte

@@ -143,6 +143,7 @@
 <section class="page">
     <SectionHeader title="Fleet" subtitle="Live health, players, and server operations.">
         {#snippet actions()}
+            <Button href="/autoscaling" variant="ghost" size="sm">Autoscaling</Button>
             <Button variant="ghost" size="sm" loading={$fleetRefreshing} onclick={() => void refreshFleet()}>Refresh</Button>
             {#if canOperate}<Button variant="primary" onclick={() => launch()}>Create server</Button>{/if}
         {/snippet}

src/routes/api/agent/operations/[id]/authorize-removal/+server.ts

@@ -0,0 +1,9 @@
+import { json } from '@sveltejs/kit';
+import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
+import { claimRemovalAuthorization } from '$lib/server/autoscaling/authorization.js';
+
+export async function POST({ request, params }: { request: Request; params: { id: string } }) {
+    const host = authenticateAgent(request);
+    try { return json(claimRemovalAuthorization(host.id, params.id, await objectBody(request, 4096))); }
+    catch (cause) { return hostApiError(cause); }
+}

src/routes/api/autoscaling/+server.ts

@@ -0,0 +1,15 @@
+import { json } from '@sveltejs/kit';
+import { previewScaling, saveScaling, scalingReport, setScalingGlobalPause, ScalingError } from '$lib/server/autoscaling/index.js';
+
+export function GET() { return json(scalingReport()); }
+export async function POST({ request }: { request: Request }) {
+    try {
+        const body = await request.json();
+        if (!body || typeof body !== 'object' || Array.isArray(body)) throw new ScalingError('Provide a policy action.');
+        if (body.id !== undefined && typeof body.id !== 'string') throw new ScalingError('Invalid policy identity.');
+        if (body.action === 'preview') return json(previewScaling(body.settings, body.id, body.revision));
+        if (body.action === 'save') return json({ policy: saveScaling(body.settings, body.id, body.revision, body.token) });
+        if (body.action === 'global-pause' && typeof body.paused === 'boolean') { setScalingGlobalPause(body.paused); return json({ ok: true }); }
+        throw new ScalingError('Choose a supported policy action.');
+    } catch (error) { return json({ error: (error as Error).message }, { status: error instanceof ScalingError ? error.status : 400 }); }
+}

src/routes/autoscaling/+page.server.ts

@@ -0,0 +1,2 @@
+import { scalingReport } from '$lib/server/autoscaling/index.js';
+export function load() { return { autoscaling: scalingReport() }; }

src/routes/autoscaling/+page.svelte

@@ -0,0 +1,136 @@
+<script lang="ts">
+    import { onMount } from 'svelte';
+    import { page } from '$app/state';
+    import { pollVisible } from '$ui/poll.js';
+    import { timeFormat } from '$ui/stores/time-zone.js';
+    import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
+    import Button from '$ui/components/primitives/Button.svelte';
+    import { zombieMaps, zombieMapBySlug } from '$lib/map-aliases.js';
+    import type { ScalingSettings, ScalingPolicy, ScalingPreview, ScalingReport } from '$lib/autoscaling.js';
+    let { data }: { data: import('./$types').PageData } = $props();
+    let snapshot = $state<ScalingReport | null>(null);
+    const report = $derived(snapshot ?? data.autoscaling);
+    const canOperate = $derived(page.data.user?.role === 'admin' || page.data.user?.role === 'operator');
+    const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'dry-run', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20, cooldownMinutes: 5 });
+    let settings = $state<ScalingSettings>(defaults());
+    let editing = $state<ScalingPolicy | null>(null);
+    let editorOpen = $state(false);
+    let preview = $state<ScalingPreview | null>(null);
+    let busy = $state(false), actionError = $state(''), refreshError = $state(''), notice = $state('');
+    let generation = 0;
+    const modeLabel = (mode: string) => ({ off: 'Off', 'dry-run': 'Dry run', 'create-only': 'Create only', 'create-destroy': 'Create and destroy' })[mode] ?? mode;
+    async function refresh(signal?: AbortSignal) {
+        const version = ++generation;
+        try {
+            const response = await fetch('/api/autoscaling', { signal });
+            if (!response.ok) throw new Error('Could not refresh policies. The last successful report is shown.');
+            const result = await response.json();
+            if (version === generation && !signal?.aborted) { snapshot = result; refreshError = ''; }
+        } catch (error) { if (version === generation && !signal?.aborted) refreshError = (error as Error).message; }
+    }
+    onMount(() => pollVisible(refresh, 15_000, false));
+    function edit(policy: ScalingPolicy | null) {
+        editing = policy; settings = policy ? { ...policy } : defaults();
+        preview = null; actionError = ''; notice = ''; editorOpen = true;
+    }
+    async function post(body: object) {
+        const response = await fetch('/api/autoscaling', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+        const result = await response.json();
+        if (!response.ok) throw new Error(result.error ?? 'Policy action failed.');
+        return result;
+    }
+    async function submit(action: 'preview' | 'save') {
+        busy = true; actionError = ''; notice = ''; ++generation;
+        try {
+            const result = await post({ action, settings, id: editing?.id, revision: editing?.revision, token: preview?.token });
+            if (action === 'preview') preview = result;
+            else { notice = 'Policy saved. Evaluation runs every 15 seconds.'; editorOpen = false; preview = null; await refresh(); }
+        } catch (error) { actionError = (error as Error).message; preview = null; }
+        finally { busy = false; }
+    }
+    async function pauseAll() {
+        busy = true; actionError = ''; ++generation;
+        try { await post({ action: 'global-pause', paused: !report.globalPaused }); notice = report.globalPaused ? 'Autoscaling resumed.' : 'Autoscaling paused. Review any unsettled operations below.'; await refresh(); }
+        catch (error) { actionError = (error as Error).message; }
+        finally { busy = false; }
+    }
+</script>
+
+<svelte:head><title>Autoscaling | Electric Cherry</title></svelte:head>
+<div class="page">
+    <SectionHeader title="Autoscaling" subtitle="Keep empty lobbies available for new groups, within the limits you choose.">
+        {#snippet actions()}<Button onclick={() => refresh()}>Refresh</Button>{#if canOperate}<Button disabled={busy} onclick={pauseAll}>{report.globalPaused ? 'Resume autoscaling' : 'Pause all'}</Button><Button variant="primary" disabled={busy} onclick={() => edit(null)}>Add policy</Button>{/if}{/snippet}
+    </SectionHeader>
+    <p class="context"><a href="/">Fleet</a> · <a href="/collections">Collections</a> · <a href="/metrics">Metrics</a></p>
+    {#if report.globalPaused}<p class="banner">All autoscaling decisions are paused. Delivered operations may still need to settle.</p>{/if}
+    <p>Start with a two-week dry run. Dry run records hypothetical actions and reserves no lobby or host capacity.</p>
+    {#if refreshError}<p role="alert">{refreshError}</p>{/if}
+    {#if actionError}<p role="alert">{actionError}</p>{/if}
+    {#if notice}<p role="status">{notice}</p>{/if}
+    {#if editorOpen && canOperate}
+        <section class="panel" aria-labelledby="policy-editor">
+            <h2 id="policy-editor">{editing ? 'Edit policy' : 'New policy'}</h2>
+            <form onsubmit={event => { event.preventDefault(); void submit('preview'); }} oninput={() => preview = null} onchange={() => preview = null}>
+                <fieldset disabled={busy}>
+                    <div class="fields">
+                        <label>Game<select bind:value={settings.game} disabled={!!editing} onchange={() => settings.mapSlug = zombieMaps(settings.game)[0].slug}><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label>
+                        <label>Map variant<select bind:value={settings.mapSlug} disabled={!!editing}>{#each zombieMaps(settings.game) as map}<option value={map.slug}>{map.friendly}</option>{/each}</select></label>
+                        <label>Mode<select bind:value={settings.mode}><option value="off">Off</option><option value="dry-run">Dry run</option>{#if report.liveAvailable}<option value="create-only">Create only</option>{/if}{#if report.removalAvailable}<option value="create-destroy">Create and destroy</option>{/if}</select></label>
+                        <label>Minimum running lobbies<input type="number" min="1" max="20" required bind:value={settings.minimum} /></label>
+                        <label>Maximum lobbies<input type="number" min="1" max="20" required bind:value={settings.maximum} /></label>
+                        <label>Empty spare target<input type="number" min="1" max="20" required bind:value={settings.spares} /></label>
+                        <label>Empty duration in minutes<input type="number" min="1" max="1440" required bind:value={settings.emptyMinutes} /></label>
+                        <label>Action cooldown in minutes<input type="number" min="1" max="1440" required bind:value={settings.cooldownMinutes} /></label>
+                    </div>
+                    <div class="actions"><Button type="submit" disabled={busy}>Preview changes</Button><Button onclick={() => editorOpen = false}>Close editor</Button></div>
+                </fieldset>
+            </form>
+            {#if preview}
+                <div class="preview" aria-live="polite">
+                    <h3>Preview · {modeLabel(preview.policy.mode)}</h3>
+                    <p>Target: {preview.decision.desired} running lobbies. {preview.snapshot.keyedNames.length} saved lobby keys. {preview.snapshot.members.length} existing or pending lobbies.</p>
+                    <p>{preview.decision.action === 'wait' ? 'Wait' : `${preview.decision.action === 'create' ? 'Create' : 'Remove'} ${preview.decision.server}`}: {preview.decision.reason}</p>
+                    {#if preview.policy.mode === 'create-only' || preview.policy.mode === 'create-destroy'}<p>Saving gives this policy exclusive control of the map's lobby count. Pause it before launching a competing collection or manual lifecycle action.</p>{/if}
+                    {#if preview.snapshot.members.length}<ul>{#each preview.snapshot.members as member}<li>{member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}</li>{/each}</ul>{/if}
+                    {#each [...preview.blockers, ...preview.snapshot.blockers] as blocker}<p>{blocker}</p>{/each}
+                    <Button variant="primary" disabled={busy || preview.blockers.length > 0} onclick={() => submit('save')}>Save reviewed policy</Button>
+                </div>
+            {/if}
+        </section>
+    {/if}
+    {#if !report.policies.length}<p class="panel">No autoscaling policies. Add a policy to observe a map in dry run.</p>{/if}
+    {#each report.policies as view (view.policy.id)}
+        <section class="panel" id={view.policy.id} aria-labelledby={`title-${view.policy.id}`}>
+            <div class="heading"><h2 id={`title-${view.policy.id}`}>{zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}</h2><span>{modeLabel(view.policy.mode)}</span>{#if canOperate}<Button disabled={busy} onclick={() => edit(view.policy)}>Edit or pause</Button>{/if}</div>
+            <p class="counts">{view.decision.occupied} occupied · {view.decision.empty} empty and ready · {view.decision.pending} pending · {view.decision.unknown} unknown or stopped</p>
+            {#if view.unsettled.length}<div class="banner"><strong>Work is still settling.</strong>{#each view.unsettled as operation}<p><a href={`/provisioning/${operation.id}`}>{operation.action} · {operation.state}</a> — {operation.stage}</p>{/each}{#if view.ownsMap}<p>This policy retains map ownership until its outstanding work settles.</p>{/if}</div>{/if}
+            <p>Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyedNames.length} saved keys</p>
+            <p><strong>{view.policy.mode === 'dry-run' ? 'Hypothetical decision: ' : 'Current decision: '}</strong>{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}</p>
+            {#if view.policy.mode === 'create-only' && view.decision.action === 'destroy'}<p>Removal is advisory in Create only mode.</p>{/if}
+            <p class="muted">Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}{view.policy.nextActionAt > Date.now() ? ` · Next action after ${$timeFormat.dateTime(new Date(view.policy.nextActionAt).toISOString())}` : ''}</p>
+            <details><summary>Lobbies and evidence</summary><ul>{#each view.snapshot.members as member}<li><a href={`/servers/${encodeURIComponent(member.name)}`}>{member.name}</a> · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}<br />{member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · <a href={`/provisioning/${member.operationId}`}>Provisioning</a>{/if}</li>{/each}</ul>{#if !view.snapshot.members.length}<p>No registered or pending lobbies.</p>{/if}{#each view.snapshot.blockers as blocker}<p>{blocker}</p>{/each}</details>
+            <details><summary>Recent decisions and measurements</summary>
+                <p>{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes</p>
+                <p>Last 31 days: {view.operations.creates} create requests · {view.operations.removals} removal requests · {view.operations.cancelled} cancelled · {view.operations.failed} failed</p>
+                {#if view.operations.readinessSamples}<p>Request to verified readiness: mean {view.operations.readinessMeanSeconds?.toFixed(0)} seconds, maximum {view.operations.readinessMaxSeconds?.toFixed(0)} seconds across {view.operations.readinessSamples} completed creates. Includes queue time.</p>{/if}
+                <p class="muted">Up to 31 days of bounded observations; gaps are excluded. Dry run measures the actual fleet, not a simulated replacement lineup. Readiness does not certify browser visibility or successful joins.</p>
+                <ol>{#each view.history as item}<li>{$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}<br />{item.decision.reason}{#if item.operationId} · <a href={`/provisioning/${item.operationId}`}>{item.operationState ?? 'View operation'}</a>{/if}</li>{/each}</ol>
+            </details>
+        </section>
+    {/each}
+</div>
+
+<style>
+    .page { max-width: 1200px; margin: 0 auto; padding: 24px; }
+    .panel { margin: 20px 0; padding: 20px; border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); background: var(--surface); overflow-wrap: anywhere; }
+    .heading, .actions { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
+    h2 { font-size: var(--text-lg); margin: 0; } h3 { font-size: var(--text-md); }
+    .heading h2 { flex: 1 1 200px; } p, li { line-height: 1.6; } .muted, .context { color: var(--text-secondary); font-size: var(--text-sm); }
+    .fields { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 220px), 1fr)); gap: 16px; margin-bottom: 20px; }
+    label { display: flex; flex-direction: column; gap: 8px; font-size: var(--text-sm); }
+    input, select { padding: 10px; min-width: 0; color: var(--text-primary); background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); font: inherit; }
+    fieldset { border: 0; padding: 0; margin: 16px 0; min-width: 0; }
+    .preview, .banner { padding: 16px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); }
+    details { margin-top: 16px; } summary { cursor: pointer; } li { margin: 10px 0; }
+    @media (max-width: 500px) { .page { padding: 16px; } .panel { padding: 16px; } }
+</style>

src/routes/collections/+page.svelte

@@ -94,7 +94,7 @@
 <svelte:head><title>Collections | Electric Cherry</title></svelte:head>
 <div class="page">
     <SectionHeader title="Collections" subtitle="Save your lineup, preview fleet changes, and follow launch progress and history.">
-        {#snippet actions()}{#if canOperate}<Button variant="primary" onclick={() => { editingBlueprint = null; builderOpen = true; }}>Create collection</Button>{/if}{/snippet}
+        {#snippet actions()}<Button href="/autoscaling">Autoscaling</Button>{#if canOperate}<Button variant="primary" onclick={() => { editingBlueprint = null; builderOpen = true; }}>Create collection</Button>{/if}{/snippet}
     </SectionHeader>
     {#if canOperate}<BlueprintBuilder bind:open={builderOpen} blueprint={editingBlueprint} onsaved={blueprintSaved} />
     <DeployCollectionDialog bind:open={collectionOpen} {blueprintId} />{/if}

src/routes/metrics/+page.svelte

@@ -108,7 +108,7 @@
 <svelte:head><title>Metrics · Electric Cherry</title></svelte:head>
 
 <div class="page">
-    <SectionHeader title="Metrics" />
+    <SectionHeader title="Metrics">{#snippet actions()}<Button href="/autoscaling">Autoscaling</Button>{/snippet}</SectionHeader>
     <div class="toolbar">
         <nav aria-label="Metrics time range">
             {#each [1, 7, 30] as days}

static/agent/ec-agent.py

@@ -33,7 +33,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.6.0"
+VERSION = "1.7.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -71,6 +71,10 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+class AutoscalingInvalidated(Rejected):
+    """This removal needs a new policy decision, never a later local retry."""
+
+
 def systemctl(*args):
     try:
         result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -1356,7 +1360,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "instance.safety-override", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1581,6 +1585,24 @@ class Agent:
             raise AgentError("Staged file checksum mismatch")
         return content
 
+    def authorize_autoscaling_removal(self, instance_id, generation, payload):
+        authorization = payload.get("autoscaling")
+        if not isinstance(authorization, dict) or not self.executing_id or payload.get("safetyOverride") or payload.get("stageId"):
+            raise AutoscalingInvalidated("Invalid autoscaling removal authorization")
+        started = time.monotonic()
+        try:
+            response = self.request("POST", "/api/agent/operations/" + urllib.parse.quote(self.executing_id, safe="") + "/authorize-removal", authorization)
+        except AgentError:
+            raise AutoscalingInvalidated("Autoscaling removal authorization unavailable; request a new decision") from None
+        ttl = response.get("validForMs") if isinstance(response, dict) else None
+        if not isinstance(response, dict) or response.get("authorized") is not True or isinstance(ttl, bool) or not isinstance(ttl, (int, float)) or not 0 < ttl <= 5000:
+            raise AutoscalingInvalidated("Autoscaling removal authorization was withdrawn or expired")
+        # Occupancy can change during the controller round trip. Check again
+        # within the short claim window, immediately before the durable checkpoint.
+        self.safe_to_disrupt(instance_id, generation)
+        if (time.monotonic() - started) * 1000 >= ttl:
+            raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
+
     def lifecycle(self, kind, instance_id, generation, payload):
         if kind == "instance.cancel-create":
             return self.cancel_creation(instance_id, generation, payload)
@@ -1610,6 +1632,8 @@ class Agent:
             # The authenticated controller records explicit administrator approval
             # for this immutable operation. Ownership and generation checks still apply.
             override_safety = payload.get("safetyOverride") is True
+            if "autoscaling" in payload and (kind != "instance.remove" or override_safety):
+                raise AutoscalingInvalidated("Autoscaling only permits removal with player safety checks")
             if kind == "instance.rebuild":
                 next_generation = payload.get("nextGeneration", payload.get("newGeneration"))
                 if isinstance(next_generation, bool) or not isinstance(next_generation, int) or next_generation <= generation:
@@ -1618,6 +1642,8 @@ class Agent:
                 self.validate_spec(spec)
             if not override_safety:
                 self.safe_to_disrupt(instance_id, generation)
+            if "autoscaling" in payload:
+                self.authorize_autoscaling_removal(instance_id, generation, payload)
             self.checkpoint({"phase": "disrupting", "containerId": health.get("containerId"), "startedAt": health.get("startedAt")})
             if payload.get("stageId") and kind == "instance.restart":
                 before = self.activate(instance_id, generation, payload["stageId"], override_safety=override_safety)
@@ -2203,7 +2229,11 @@ class Agent:
         try:
             result = self.dispatch(operation)
         except Held as err:
+            if operation.get("kind") == "instance.remove" and "autoscaling" in operation.get("payload", {}):
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
             return self.journal.update(operation["id"], "accepted", {"blocked": "occupied" if "Players" in str(err) else "unknown", "message": str(err)})
+        except AutoscalingInvalidated as err:
+            return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error=str(err))
         except Exception as err:
             # The command may already have changed Docker or files. Never infer
             # failure means nothing happened, and never retry without inspection.
@@ -2239,6 +2269,8 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
+                return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":
                 return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
             if kind == "instance.create":

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.6.0",
+  "version": "1.7.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "bcc0a207125f2a1ad754c294decf2238064995ba93183b845a589a41a41925f6",
+  "sha256": "4b40206f477dcf35f0506281b3dddca32b57214a964bcf5169c8611dc705132d",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/autoscaling-dry-run.test.ts

@@ -0,0 +1,55 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
+const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' };
+
+it('persists deduplicated dry-run decisions across restart without provisioning or reservations', async () => {
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_3', 'secret');
+    const preview = scaling.previewScaling(settings);
+    const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
+    await scaling.runAutoscaling(); await scaling.runAutoscaling();
+    closeDb();
+    await scaling.runAutoscaling();
+    const view = scaling.scalingReport().policies[0];
+    expect(view.policy.id).toBe(saved.id);
+    expect(view.history).toHaveLength(1);
+    expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_3' });
+    expect(JSON.stringify(view)).not.toContain('secret');
+    for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 });
+});
+
+it('rejects duplicate policies, stale edits and changed previews', async () => {
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const preview = scaling.previewScaling(settings);
+    const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
+    expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists');
+    expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
+    expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
+    expect(() => scaling.saveScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision).token)).toThrow('Save keys');
+});
+
+it('pauses evaluations and prunes expired observations without deleting current history', async () => {
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
+    await scaling.runAutoscaling();
+    scaling.setScalingGlobalPause(true); await scaling.runAutoscaling();
+    expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused');
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const id = scaling.scalingReport().policies[0].history[0].id;
+    db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id);
+    scaling.pruneAutoscaling();
+    expect(scaling.scalingReport().policies[0].history).toHaveLength(1);
+});
+
+it('uses fleet authorization for page, reads, and mutations', () => {
+    expect(routePermission('/autoscaling', 'GET')).toBe('viewer');
+    expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer');
+    expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false);
+    expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true);
+});

tests/unit/autoscaling-live.test.ts

@@ -0,0 +1,103 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-only' };
+async function setup() {
+    await prepareNativeHost();
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    for (const n of [3, 8]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
+    const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
+    return { db, scaling, saved };
+}
+
+it('admits one durable keyed create and reuses it across ticks and restart', async () => {
+    const { db, scaling, saved } = await setup();
+    expect(await scaling.runAutoscaling()).toBe(true);
+    await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
+    const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
+    await scaling.runAutoscaling();
+    expect(db().prepare('SELECT server,state FROM server_deployments WHERE action=\'create\'').all()).toEqual([{ server: 't6_zombie_town_3', state: 'queued' }]);
+    expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 1 });
+    const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
+    const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+    expect(scalingAdmission(id)).toBeNull();
+    const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+    expect(getDeployment(id)?.sources).toContainEqual(expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` }));
+});
+
+it('requires key coverage and withdraws an unsubmitted create when the policy pauses', async () => {
+    const { db, scaling, saved } = await setup();
+    await scaling.runAutoscaling();
+    const off = { ...settings, mode: 'off' };
+    scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
+    expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' });
+    expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 0 });
+    db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run();
+    const current = scaling.getScalingPolicy(saved.id)!;
+    expect(scaling.previewScaling(settings, saved.id, current.revision).blockers.join(' ')).toContain('Save keys');
+});
+
+it('blocks manual lifecycle and collection creation, including stale previews', async () => {
+    const { db, scaling, saved } = await setup();
+    const { queueCreateServer, destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+    const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+    expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
+    expect(destroyServer('t6_zombie_town_3')).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
+    expect(requestLifecycle('t6_zombie_town_3', 'stop')).toMatchObject({ ok: false, status: 409 });
+    const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
+    const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
+    const blueprint = saveBlueprint({ name: 'Town', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] });
+    expect(previewCollection(blueprint.id).blockers.join(' ')).toContain('Autoscaling');
+    const off = { ...settings, mode: 'off' }; scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
+    const preview = previewCollection(blueprint.id);
+    const current = scaling.getScalingPolicy(saved.id)!;
+    scaling.saveScaling(settings, saved.id, current.revision, scaling.previewScaling(settings, saved.id, current.revision).token);
+    expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
+    expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 });
+});
+
+it('operator cancellation pauses the policy and does not immediately create a replacement', async () => {
+    const { db, scaling } = await setup();
+    await scaling.runAutoscaling();
+    const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
+    const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
+    expect(cancelProvisioningJob(id)).toBe(true);
+    await scaling.runAutoscaling();
+    expect(scaling.scalingReport().policies[0].policy.mode).toBe('off');
+    expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()).toEqual({ n: 1 });
+});
+
+it('exact matching cannot remove an owned map absent from the collection', async () => {
+    const town = 't6_zombie_town_1';
+    await prepareNativeHost([town]);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const at = new Date().toISOString();
+    db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at);
+    for (const n of [1, 2]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
+    const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
+    const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
+    const collection = saveBlueprint({ name: 'Moon only', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
+    const preview = previewCollection(collection.id, 'exact');
+    expect(preview.removals.map(r => r.name)).toContain(town);
+    expect(preview.blockers.join(' ')).toContain('Autoscaling');
+    expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
+});
+
+it('rechecks evidence and global pause before dispatch', async () => {
+    const { db, scaling } = await setup();
+    await scaling.runAutoscaling();
+    const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
+    const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+    db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_3'").run();
+    expect(scalingAdmission(id)).toContain('no longer eligible');
+    scaling.setScalingGlobalPause(true);
+    expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' });
+    expect(await scaling.runAutoscaling()).toBe(false);
+});

tests/unit/autoscaling-planner.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from 'vitest';
+import { scalingSettings, type ScalingMember, type ScalingPolicy, type ScalingSnapshot } from '../../src/lib/autoscaling.js';
+import { emptyObservation, planScaling } from '../../src/lib/server/autoscaling/planner.js';
+import { mapIdentity } from '../../src/lib/server/autoscaling/snapshot.js';
+
+const now = 2_000_000;
+const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
+const name = (n: number) => `t6_zombie_town_${n}`;
+const member = (n: number, state: ScalingMember['state'] = 'empty'): ScalingMember => ({ name: name(n), identity: `instance:${n}`, state, players: state === 'occupied' ? 1 : state === 'empty' ? 0 : null, observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null });
+const snapshot = (...members: ScalingMember[]): ScalingSnapshot => ({ members, reservedNames: members.map(m => m.name), keyedNames: [1, 2, 3, 4].map(name), blockers: [] });
+
+describe('autoscaling policy planning', () => {
+    it('requires explicit bounded maximum and valid variant', () => {
+        for (const input of [{}, { game: 't6', mapSlug: 'town' }, { game: 't5', mapSlug: 'town', maximum: 4 }, { game: 't6', mapSlug: 'town', maximum: 21 }, { ...policy, minimum: 5 }, { ...policy, spares: 0 }, { ...policy, emptyMinutes: 1.5 }]) expect(() => scalingSettings(input)).toThrow();
+        expect(scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4 }).mode).toBe('off');
+    });
+    it('creates a spare even with free player slots and bootstraps an empty map', () => {
+        expect(planScaling(policy, snapshot(member(1, 'occupied')), now)).toMatchObject({ action: 'create', server: name(2), desired: 2 });
+        expect(planScaling(policy, snapshot(), now)).toMatchObject({ action: 'create', desired: 1 });
+        expect(planScaling(policy, snapshot(member(1, 'occupied'), member(2)), now).action).toBe('wait');
+    });
+    it('uses nonconsecutive keyed identities without moving keys', () => {
+        expect(planScaling(policy, { ...snapshot(), keyedNames: [name(8), name(3)] }, now).server).toBe(name(3));
+        expect(planScaling(policy, { ...snapshot(), keyedNames: [] }, now).reason).toContain('saved key');
+    });
+    it('holds for pending creation, unknown, stopped, or protected members', () => {
+        for (const state of ['creating', 'removing', 'unknown', 'stopped', 'protected'] as const) expect(planScaling(policy, snapshot(member(1, state)), now).action).toBe('wait');
+        expect(planScaling(policy, { ...snapshot(), blockers: ['Unresolved instance'] }, now).action).toBe('wait');
+    });
+    it('holds for stale and future samples, cooldown, and disabled policies', () => {
+        for (const stamp of [null, now - 90_001, now + 1]) expect(planScaling(policy, snapshot({ ...member(1, 'occupied'), observedAt: stamp }), now).action).toBe('wait');
+        expect(planScaling({ ...policy, nextActionAt: now + 1 }, snapshot(), now).action).toBe('wait');
+        expect(planScaling({ ...policy, mode: 'off' }, snapshot(), now).action).toBe('wait');
+    });
+    it('never sacrifices occupied lobbies or minimum/spare capacity to a maximum', () => {
+        expect(planScaling(policy, snapshot(...[1, 2, 3, 4].map(n => member(n, 'occupied'))), now)).toMatchObject({ action: 'wait', desired: 4 });
+        expect(planScaling({ ...policy, maximum: 1 }, snapshot(member(1, 'occupied'), member(2)), now).action).toBe('wait');
+        expect(planScaling({ ...policy, minimum: 3 }, snapshot(member(1), member(2), member(3)), now).action).toBe('wait');
+    });
+    it('trims one longest-empty lobby and requires a full observed interval', () => {
+        expect(planScaling(policy, snapshot(member(1, 'occupied'), member(2), member(3), member(4)), now)).toMatchObject({ action: 'destroy', server: name(2), desired: 2 });
+        expect(planScaling(policy, snapshot({ ...member(1), emptySince: now }, { ...member(2), emptySince: now }), now).action).toBe('wait');
+    });
+    it('preserves map variants and refuses ambiguous engine maps', () => {
+        expect(mapIdentity({ name: 'legacy', game: 't6', map: 'zm_transit' })).toBeNull();
+        expect(mapIdentity({ name: name(1), map: 'zm_transit' })?.mapSlug).toBe('town');
+        expect(mapIdentity({ name: 'legacy', game: 't6', map_slug: 'farm', map: 'zm_transit' })?.mapSlug).toBe('farm');
+    });
+});
+
+describe('continuous empty evidence', () => {
+    it('does not turn rereads into new evidence', () => {
+        const previous = emptyObservation(member(1), undefined, now);
+        const reread = emptyObservation(member(1), previous, now + 60_000);
+        expect(reread).toMatchObject({ observedAt: now, emptySince: now });
+        expect(emptyObservation({ ...member(1), observedAt: now + 60_000 }, previous, now + 60_000).emptySince).toBe(now);
+    });
+    it('resets on unknown, players, lifecycle replacement, and observation gaps', () => {
+        const previous = emptyObservation(member(1), undefined, now);
+        expect(emptyObservation(member(1, 'occupied'), previous, now).emptySince).toBeNull();
+        expect(emptyObservation(member(1, 'unknown'), previous, now).emptySince).toBeNull();
+        expect(emptyObservation({ ...member(1), identity: 'replacement', observedAt: now + 1 }, previous, now + 1).emptySince).toBe(now + 1);
+        expect(emptyObservation({ ...member(1), observedAt: now + 120_000 }, previous, now + 120_000).emptySince).toBe(now + 120_000);
+        expect(emptyObservation({ ...member(1), emptyStartedAt: now + 10_000, observedAt: now + 15_000 }, previous, now + 15_000).emptySince).toBe(now + 15_000);
+    });
+});

tests/unit/autoscaling-removal.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0) }));
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); });
+const names = ['t6_zombie_town_1', 't6_zombie_town_2'];
+const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy', emptyMinutes: 1, cooldownMinutes: 1 };
+
+async function setup() {
+    const host = await prepareNativeHost(names);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const authorization = await import('../../src/lib/server/autoscaling/authorization.js');
+    db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?").run(JSON.stringify([authorization.autoscalingRemovalCapability]), host.host.id);
+    const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base);
+    for (const name of names) {
+        db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,last_players) VALUES(?,'t6','zm_transit','town','running',0)").run(name);
+        db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(name, `key-${name}`);
+    }
+    const fresh = (offset: number) => {
+        vi.setSystemTime(base + offset); const at = new Date().toISOString();
+        db().prepare('UPDATE servers SET container_checked_at=?,last_seen_at=?,last_checked_at=?').run(at, at, at);
+        db().prepare('UPDATE ec_hosts SET last_heartbeat=?').run(at);
+    };
+    fresh(0);
+    const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
+    for (const offset of [0, 30_000, 60_000]) { fresh(offset); await scaling.runAutoscaling(); }
+    const intent = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
+    expect(intent?.operation_id).toBeTruthy();
+    const backend = await import('../../src/lib/server/backends/native.js');
+    vi.spyOn(backend, 'nativeRequest').mockResolvedValue({ healthy: true, state: 'running', exists: true } as never);
+    const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+    const dispatch = async () => {
+        await processProvisioning();
+        const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(intent.operation_id) as { operation_id: string };
+        expect(row.operation_id).toBeTruthy();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const command = getHostOperation(row.operation_id)!;
+        return { id: row.operation_id, payload: command.payload.autoscaling as Record<string, unknown> };
+    };
+    return { db, scaling, authorization, policy: saved, operationId: intent.operation_id, hostId: host.host.id, fresh, dispatch, processProvisioning };
+}
+
+it('removes only after a full observed empty interval and claims authorization once', async () => {
+    const s = await setup();
+    const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+    expect(enableSafetyOverride(s.operationId, { id: 'admin', username: 'admin' }).ok).toBe(false);
+    const command = await s.dispatch();
+    expect(s.authorization.claimRemovalAuthorization('wrong-host', command.id, command.payload).authorized).toBe(false);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload)).toMatchObject({ authorized: true, validForMs: 5000 });
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+});
+
+it('withdraws a queued removal when a player arrives and restarts the empty interval', async () => {
+    const s = await setup();
+    s.db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(names[0]);
+    await s.scaling.runAutoscaling();
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' });
+    s.db().prepare('UPDATE servers SET last_players=0').run();
+    s.fresh(90_000); await s.scaling.runAutoscaling();
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 0 });
+});
+
+it('rechecks controller occupancy after the async probe even before monitoring refreshes', async () => {
+    const s = await setup();
+    const rcon = await import('../../src/lib/server/integrations/rcon.js');
+    vi.mocked(rcon.rconGetPlayerCount).mockResolvedValueOnce(1);
+    await s.processProvisioning();
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' });
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_authorizations').get()).toEqual({ n: 0 });
+});
+
+it('revokes a delivered removal on policy pause without claiming it was undone', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    s.db().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(new Date().toISOString(), command.id);
+    const off = { ...settings, mode: 'off' };
+    s.scaling.saveScaling(off, s.policy.id, s.policy.revision, s.scaling.previewScaling(off, s.policy.id, s.policy.revision).token);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+    expect(s.db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 1 });
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'running' });
+    expect(s.scaling.scalingReport().policies[0]).toMatchObject({ ownsMap: true, unsettled: [{ id: s.operationId, state: 'running' }] });
+});
+
+it('withdraws undelivered commands and reconciles invalidation without a retry or failure pause', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    s.scaling.setScalingGlobalPause(true);
+    expect(s.db().prepare('SELECT state FROM ec_host_operations WHERE id=?').get(command.id)).toEqual({ state: 'failed' });
+    await s.processProvisioning(); await s.scaling.runAutoscaling();
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' });
+    expect(s.scaling.getScalingPolicy(s.policy.id)?.pausedReason).toBeNull();
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 0 });
+});
+
+it('rejects expired, changed revision, and newly protected removal authorizations', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, { ...command.payload, revision: 999 }).authorized).toBe(false);
+    s.db().prepare("INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)").run(names[1], new Date().toISOString());
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+    s.db().prepare('DELETE FROM maintenance_jobs').run();
+    s.fresh(130_000);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+});

tests/unit/autoscaling-snapshot.test.ts

@@ -0,0 +1,36 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+import { scalingSettings, type ScalingPolicy } from '../../src/lib/autoscaling.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+const town = 't6_zombie_town_1', farm = 't6_zombie_farm_1';
+const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
+
+it('separates variants, deduplicates pending identities, and initializes additive schema', async () => {
+    await prepareNativeHost([town, farm]);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
+    const { ensureAutoscalingSchema } = await import('../../src/lib/server/autoscaling/schema.js');
+    ensureAutoscalingSchema(db()); ensureAutoscalingSchema(db());
+    const at = new Date().toISOString();
+    for (const server of [town, farm]) db().prepare(`INSERT INTO servers(name,container_state,container_checked_at,last_seen_at,last_checked_at,last_players) VALUES(?,'running',?,?,?,0)`).run(server, at, at, at);
+    expect(scalingSnapshot(policy).members.map(m => [m.name, m.state])).toEqual([[town, 'empty']]);
+    db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES('create',?,'create','queued','Awaiting placement',?,'test')`).run(town, at);
+    db().prepare(`INSERT INTO server_creation_jobs(deployment_id,server,game_mode,map_slug,expected_map) VALUES('create',?,'zombie','town','zm_transit')`).run(town);
+    const snapshot = scalingSnapshot(policy);
+    expect(snapshot.members).toHaveLength(1);
+    expect(snapshot.members[0].state).toBe('creating');
+    expect(snapshot.reservedNames).toEqual([town]);
+});
+
+it('does not create schema mutations or host work when planning an empty map', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
+    const { planScaling } = await import('../../src/lib/server/autoscaling/planner.js');
+    db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_8', 'private-key');
+    expect(planScaling(policy, scalingSnapshot(policy), Date.now()).server).toBe('t6_zombie_town_8');
+    expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 });
+});