Allow rebuilds on autoscaling-managed servers

docs/guides/autoscaling.md

@@ -25,7 +25,7 @@ Existing policies saved in the removed Dry run or Create only modes switch to Of
 
 ## Ownership and blockers
 
-A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. Competing collection launches and manual Create, Start, Stop, Rebuild, or Destroy requests are rejected with a policy link. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish.
+A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. Competing collection launches and manual Create, Start, Stop, or Destroy requests are rejected with a policy link. Restart and Rebuild remain available through the usual provisioning checks. Their pending work and unresolved recovery hold scaling decisions for the affected map, so temporary downtime does not trigger replacement lobbies. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish.
 
 Unknown or stale occupancy, stopped or unhealthy members, active betas, campaigns, maintenance, host drains, and unresolved recovery hold the policy's decisions. Autoscaling does not repair these conditions itself. The page and linked Provisioning operation explain the current blocker, including capacity and game key waits.
 

src/lib/server/provisioning/requests.ts

@@ -121,7 +121,8 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
 
 export function requestLifecycle(server: string, action: RuntimeAction) {
     if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
-    const owned = action !== 'restart' ? scalingOwnershipBlocker(server) : null;
+    // Restart and rebuild preserve the lineup; their pending work holds scaling decisions.
+    const owned = action === 'start' || action === 'stop' ? scalingOwnershipBlocker(server) : null;
     if (owned) return { ok: false as const, status: 409, error: owned };
     if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
     return db().transaction(() => {

tests/unit/autoscaling-lifecycle.test.ts

@@ -0,0 +1,84 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+import { zombieMaps } from '../../src/lib/map-aliases.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+const server = 't6_zombie_town_1';
+
+async function setup(mapSlug: string) {
+    const maps = zombieMaps('t6').filter(map => mapSlug === '*' || map.slug === mapSlug);
+    const names = maps.map(map => `t6_zombie_${map.slug}_1`);
+    const { host } = await prepareNativeHost(names);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js');
+    db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?")
+        .run(JSON.stringify([autoscalingRemovalCapability]), host.id);
+    const at = new Date().toISOString();
+    for (const map of maps) {
+        const name = `t6_zombie_${map.slug}_1`;
+        db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
+            VALUES(?,'t6',?,?,'running',?,?,?,0)`).run(name, map.canonical, map.slug, at, at, at);
+        db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(name, `key-${name}`);
+        db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name);
+    }
+    // A free key ensures a lack of capacity cannot hide unwanted scale-up.
+    db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')").run();
+    const policy = scaling.saveScaling({ game: 't6', mapSlug, maximum: 2, mode: 'create-destroy' });
+    const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+    return { db, scaling, policy, requestLifecycle };
+}
+
+it.each(['town', '*'])('keeps lineup-changing actions blocked for %s ownership', async scope => {
+    const { db, requestLifecycle } = await setup(scope);
+    const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+    for (const action of ['start', 'stop'] as const) {
+        expect(requestLifecycle(server, action)).toMatchObject({ ok: false, status: 409, error: expect.stringContaining('Autoscaling') });
+    }
+    expect(destroyServer(server)).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+});
+
+it.each([
+    ['town', 'restart'], ['town', 'rebuild'], ['*', 'restart'], ['*', 'rebuild']
+] as const)('allows %s-owned %s without scaling up during work or recovery', async (scope, action) => {
+    const { db, scaling, policy, requestLifecycle } = await setup(scope);
+    const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
+    const result = requestLifecycle(server, action);
+    expect(result.ok).toBe(true);
+    if (!result.ok) throw new Error(result.error);
+    expect(requestLifecycle(server, action)).toEqual(result);
+
+    for (const state of ['waiting', 'queued', 'running', 'awaiting-health', 'interrupted']) {
+        db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, result.deploymentId);
+        db().prepare("UPDATE servers SET container_state='stopped',last_players=NULL WHERE name=?").run(server);
+        const snapshot = scalingSnapshot(policy);
+        expect(snapshot.members.find(member => member.name === server)).toMatchObject({ state: 'protected', operationId: result.deploymentId });
+        expect(snapshot.reservedNames).toContain(server);
+        expect(await scaling.runAutoscaling()).toBe(false);
+    }
+
+    // A terminal operation with unresolved recovery must still hold capacity.
+    db().prepare("UPDATE server_deployments SET state='failed' WHERE id=?").run(result.deploymentId);
+    db().prepare('INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?)')
+        .run(server, new Date().toISOString(), result.deploymentId);
+    expect(scalingSnapshot(policy).members.find(member => member.name === server)?.blocker).toBe('Unresolved recovery');
+    expect(await scaling.runAutoscaling()).toBe(false);
+    expect(db().prepare("SELECT id FROM server_deployments WHERE action IN ('create','destroy')").all()).toEqual([]);
+
+    const at = new Date().toISOString();
+    db().prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id=?").run(at, result.deploymentId);
+    db().prepare('UPDATE restart_rollout SET healthy_at=? WHERE server=?').run(at, server);
+    db().prepare("UPDATE servers SET container_state='running',last_players=0,container_checked_at=?,last_seen_at=?,last_checked_at=?")
+        .run(at, at, at);
+    expect(await scaling.runAutoscaling()).toBe(false);
+    expect(scaling.getScalingPolicy(policy.id)).toMatchObject({ mode: 'create-destroy', pausedReason: null });
+
+    // Once recovered, actual occupancy can request a new spare normally.
+    db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(server);
+    expect(await scaling.runAutoscaling()).toBe(true);
+    expect(db().prepare("SELECT server FROM server_deployments WHERE action='create'").all()).toEqual([{ server: 't6_zombie_town_2' }]);
+});