import { createHash, randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js'; import { emptyObservation, planScaling } from './planner.js'; import { parseServerName } from '../../map-aliases.js'; import { scalingSnapshot } from './snapshot.js'; import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js'; import { cancelCreation } from '../provisioning/cancel-creation.js'; import { scalingAdmission } from './admission.js'; import { destroyServer } from '../provisioning/instances.js'; import { cancelRemoval } from '../provisioning/history.js'; import { autoscalingRemovalCapability, revokePolicyRemovals } from './authorization.js'; import { getHost } from '../hosts/index.js'; import { scalingOwner } from './ownership.js'; import { getServer } from '../fleet/servers.js'; export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } } interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string } const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, nextActionAt: row.next_action_at, pausedReason: row.paused_reason, updatedAt: row.updated_at }); export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present); export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); } export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1'; export function previewScaling(input: unknown, id?: string, revision?: number): ScalingPreview { const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined; if (id && !existing) throw new ScalingError('Policy not found.', 404); if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409); if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.'); const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0, nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' }; const snapshot = scalingSnapshot(policy); const blockers: string[] = []; if (liveScaling(policy.mode)) { const owner = scalingOwner(policy.game, policy.mapSlug); if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`); blockers.push(...snapshot.blockers); for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`); if (policy.mode === 'create-destroy') for (const member of snapshot.members) { const hostId = getServer(member.name)?.host_id; if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`); } if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.'); try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); } } const decision = planScaling(policy, snapshot, Date.now()); const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(), members: snapshot.members.map(m => [m.name, m.identity, m.state]), keyPool: snapshot.keyPool, reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex'); return { policy, snapshot, decision, token, blockers }; } export function saveScaling(input: unknown, id?: string, revision?: number, token?: string): ScalingPolicy { return db().transaction(() => { const preview = previewScaling(input, id, revision); if (!token || preview.token !== token) throw new ScalingError('The policy preview changed. Review it again before saving.', 409); if (preview.blockers.length) throw new ScalingError(preview.blockers[0]); const policy = preview.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString(); if (id) revokePolicyRemovals(id); if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409); db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at); // An edit starts a new observation interval; earlier inactivity is not authorization. db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId); db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId); if (liveScaling(policy.mode)) for (const member of preview.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)') .run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() })); reconcileScaling(getScalingPolicy(nextId)!); return getScalingPolicy(nextId)!; })(); } export function deleteScaling(id: string, revision: number): void { db().transaction(() => { const policy = getScalingPolicy(id); if (!policy) throw new ScalingError('Policy not found.', 404); if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409); // Keep admission evidence and ownership while a host command may still act. if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) { throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409); } revokePolicyRemovals(id); db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id); db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(id); db().prepare('DELETE FROM autoscaling_samples WHERE policy_id=?').run(id); db().prepare('DELETE FROM autoscaling_decisions WHERE policy_id=?').run(id); db().prepare('DELETE FROM autoscaling_policies WHERE id=?').run(id); })(); } export function setScalingGlobalPause(paused: boolean): void { db().transaction(() => { db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0'); if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); } })(); } function reconcileScaling(policy: ScalingPolicy): void { const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as { id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[]; for (const intent of intents) { if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at && (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false); if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) { revokePolicyRemovals(policy.id); if (!intent.command_id) cancelRemoval(intent.operation_id, false); } const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null }; const terminal = ['done','warning','failed','cancelled'].includes(current.state); const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined; const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true; const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed' && !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id); const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending; if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) { db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id); } if (terminal && settled) { db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id); if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id); } } if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) { db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id); } } export async function runAutoscaling(): Promise { const now = Date.now(); let submitted = false; for (const initial of listScalingPolicies()) { reconcileScaling(initial); const policy = getScalingPolicy(initial.id)!; if (policy.mode === 'off') continue; db().transaction(() => { const snapshot = scalingSnapshot(policy, now); if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined; if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`); const decision = planScaling(policy, snapshot, now); for (const member of snapshot.members) { // Keep the observation interval that justified a pending removal; // admission reads fresh evidence and revokes it on any new blocker. if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue; const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now); db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?) ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation)); } // Evidence includes provenance and timestamps but never keys or player rosters. const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers }); const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex'); const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined; const execute = liveScaling(policy.mode) && (decision.action === 'create' || policy.mode === 'create-destroy' && decision.action === 'destroy'); const decisionId = randomUUID(); if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)') .run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence); if (execute) { db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId); try { db().transaction(() => { const result = decision.action === 'create' ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug, selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId }) : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId); if (!result.ok) throw new Error(result.message); if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.'); db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId); db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server); db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id); submitted = true; })(); } catch (error) { db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId); db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id); } } db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`) .run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit); })(); } return submitted; } export function pruneAutoscaling(now = Date.now()): void { const cutoff = now - 31 * 86400_000; db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at { const snapshot = scalingSnapshot(policy); if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); const decision = planScaling(policy, snapshot, Date.now()); const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[]) .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state })); const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled']; const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map; // Integrate only bounded intervals between adjacent samples; outages contribute no invented time. const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes, COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours, COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes, COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals']; const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals, COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed, COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples, AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds, MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations']; return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap }; }); return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true }; }