Require revocable authorization for automatic lobby removal

src/lib/server/autoscaling/admission.ts

@@ -19,6 +19,7 @@ export function scalingAdmission(operationId: string): string | null {
     else {
         const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' };
         const requested = JSON.parse(intent.decision) as ScalingDecision;
+        if (requested.action === 'destroy' && row.mode !== 'create-destroy') return 'Automatic removal is disabled.';
         const snapshot = scalingSnapshot(policy, Date.now(), operationId);
         if (requested.action === 'create') {
             snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);

src/lib/server/autoscaling/authorization.ts

@@ -0,0 +1,52 @@
+import { db } from '../db/index.js';
+import { scalingAdmission, scalingIntent } from './admission.js';
+import { getHost } from '../hosts/index.js';
+
+export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
+/** Authorization is attached to the immutable host command inside its submission transaction. */
+export function removalAuthorization(operationId: string, hostId: string): object | null {
+    const intent = scalingIntent(operationId);
+    if (!intent) return null;
+    if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
+    const reason = scalingAdmission(operationId);
+    if (reason) throw new Error(reason);
+    const expiresAt = Date.now() + 60_000;
+    db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
+        .run(operationId, intent.policy_id, intent.revision, expiresAt);
+    return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
+}
+
+/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
+export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
+    return db().transaction(() => {
+        const now = Date.now();
+        const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
+            JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
+            JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+            JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
+            WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
+            { operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
+        if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
+            || row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
+            || row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
+        const reason = scalingAdmission(row.operation_id);
+        if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
+        db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
+        // Only a short final occupancy check may occur after this boundary.
+        return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
+    })();
+}
+
+export function revokePolicyRemovals(policyId: string): void {
+    if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
+    db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
+    const at = new Date().toISOString();
+    const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
+        JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
+        WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
+    for (const command of commands) {
+        const result = JSON.stringify({ autoscalingInvalidated: true });
+        db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
+        db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
+    }
+}

src/lib/server/autoscaling/index.ts

@@ -6,6 +6,11 @@ import { scalingSnapshot } from './snapshot.js';
 import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
 import { cancelCreation } from '../provisioning/cancel-creation.js';
 import { scalingAdmission } from './admission.js';
+import { destroyServer } from '../provisioning/instances.js';
+import { cancelRemoval } from '../provisioning/history.js';
+import { autoscalingRemovalCapability, revokePolicyRemovals } from './authorization.js';
+import { getHost } from '../hosts/index.js';
+import { getServer } from '../fleet/servers.js';
 
 export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
 interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
@@ -23,11 +28,14 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
         nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
     const snapshot = scalingSnapshot(policy);
     const blockers: string[] = [];
-    if (policy.mode === 'create-destroy') blockers.push('Automatic removal is not available yet.');
     if (liveScaling(policy.mode)) {
         if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`);
         blockers.push(...snapshot.blockers);
         for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
+        if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
+            const hostId = getServer(member.name)?.host_id;
+            if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
+        }
         if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
         try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
     }
@@ -43,6 +51,7 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke
         if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409);
         if (preview.blockers.length) throw new ScalingError(preview.blockers[0]);
         const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
+        if (id) revokePolicyRemovals(id);
         if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this map.', 409);
         db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
             ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
@@ -59,7 +68,7 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke
 export function setScalingGlobalPause(paused: boolean): void {
     db().transaction(() => {
         db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
-        if (paused) for (const policy of listScalingPolicies()) reconcileScaling(policy);
+        if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
     })();
 }
 
@@ -69,18 +78,26 @@ function reconcileScaling(policy: ScalingPolicy): void {
         LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
         { id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
     for (const intent of intents) {
-        if (intent.action === 'create' && !intent.command_id && !intent.submitted_at
+        if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
             && (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
+        if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
+            revokePolicyRemovals(policy.id);
+            if (!intent.command_id) cancelRemoval(intent.operation_id, false);
+        }
         const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
         const terminal = ['done','warning','failed','cancelled'].includes(current.state);
-        const command = intent.command_id ? db().prepare('SELECT state FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string } | undefined : undefined;
+        const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
+        const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
         const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
             && !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
         const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
-        if ((current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
+        if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
             db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
         }
-        if (terminal && settled) db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
+        if (terminal && settled) {
+            db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
+            if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
+        }
     }
     if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
         db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
@@ -100,6 +117,9 @@ export async function runAutoscaling(): Promise<boolean> {
             if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.');
             const decision = planScaling(policy, snapshot, now);
             for (const member of snapshot.members) {
+                // Keep the observation interval that justified a pending removal;
+                // admission reads fresh evidence and revokes it on any new blocker.
+                if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
                 const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
                 db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
                     ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
@@ -108,7 +128,7 @@ export async function runAutoscaling(): Promise<boolean> {
             const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
             const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
             const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
-            const execute = liveScaling(policy.mode) && decision.action === 'create';
+            const execute = liveScaling(policy.mode) && (decision.action === 'create' || policy.mode === 'create-destroy' && decision.action === 'destroy');
             const decisionId = randomUUID();
             if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
             else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
@@ -117,10 +137,12 @@ export async function runAutoscaling(): Promise<boolean> {
                 db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
                 try {
                     db().transaction(() => {
-                    const selection = nativeLaunchSelection({ automatic: true }, policy.game);
-                    const result = queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug, selection,
-                        createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId });
+                    const result = decision.action === 'create'
+                        ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
+                            selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
+                        : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
                     if (!result.ok) throw new Error(result.message);
+                    if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
                     db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
                     db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
                     db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
@@ -128,7 +150,7 @@ export async function runAutoscaling(): Promise<boolean> {
                     })();
                 } catch (error) {
                     db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
-                    db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Creation could not be queued: ${(error as Error).message}`, policy.id);
+                    db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
                 }
             }
             db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
@@ -160,5 +182,5 @@ export function scalingReport(): ScalingReport {
             COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
         return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
     });
-    return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: false };
+    return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
 }

src/lib/server/autoscaling/planner.ts

@@ -7,6 +7,7 @@ export function emptyObservation(member: ScalingMember, previous: EmptyObservati
     const stamp = member.observedAt;
     const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
     const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
+        && (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
         && previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
         && now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
     return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };

src/lib/server/autoscaling/snapshot.ts

@@ -53,7 +53,9 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
         else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id} is ${operation.state}`; }
         else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
         else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
-        return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null, blocker, operationId: operation?.id ?? null };
+        const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
+        return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
+            emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
     });
     for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: 'Creation is pending.', operationId: operations.find(o => o.server === row.name)?.id ?? null });
     const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];