Require revocable authorization for automatic lobby removal

src/lib/server/autoscaling/authorization.ts

@@ -0,0 +1,52 @@
+import { db } from '../db/index.js';
+import { scalingAdmission, scalingIntent } from './admission.js';
+import { getHost } from '../hosts/index.js';
+
+export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
+/** Authorization is attached to the immutable host command inside its submission transaction. */
+export function removalAuthorization(operationId: string, hostId: string): object | null {
+    const intent = scalingIntent(operationId);
+    if (!intent) return null;
+    if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
+    const reason = scalingAdmission(operationId);
+    if (reason) throw new Error(reason);
+    const expiresAt = Date.now() + 60_000;
+    db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
+        .run(operationId, intent.policy_id, intent.revision, expiresAt);
+    return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
+}
+
+/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
+export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
+    return db().transaction(() => {
+        const now = Date.now();
+        const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
+            JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
+            JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+            JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
+            WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
+            { operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
+        if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
+            || row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
+            || row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
+        const reason = scalingAdmission(row.operation_id);
+        if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
+        db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
+        // Only a short final occupancy check may occur after this boundary.
+        return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
+    })();
+}
+
+export function revokePolicyRemovals(policyId: string): void {
+    if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
+    db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
+    const at = new Date().toISOString();
+    const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
+        JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
+        WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
+    for (const command of commands) {
+        const result = JSON.stringify({ autoscalingInvalidated: true });
+        db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
+        db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
+    }
+}