From e012278bd98eacfee677a4c5aa707382efcf9be0 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 22:59:17 -0400 Subject: [PATCH] Add exclusive map ownership and capacity-aware autoscaling creation --- src/lib/provisioning-detail.ts | 2 +- src/lib/server/autoscaling/admission.ts | 32 ++++++ src/lib/server/autoscaling/index.ts | 81 ++++++++++++-- src/lib/server/autoscaling/ownership.ts | 25 +++++ src/lib/server/db/index.ts | 2 +- src/lib/server/db/schema.sql | 2 +- src/lib/server/fleet/collections.ts | 4 + .../server/provisioning/cancel-creation.ts | 4 +- src/lib/server/provisioning/cancellation.ts | 2 + src/lib/server/provisioning/history.ts | 5 +- src/lib/server/provisioning/instances.ts | 3 + src/lib/server/provisioning/native.ts | 13 ++- src/lib/server/provisioning/requests.ts | 3 + src/lib/server/provisioning/worker.ts | 3 + src/lib/ui/provisioning-presentation.ts | 2 +- tests/unit/autoscaling-dry-run.test.ts | 2 +- tests/unit/autoscaling-live.test.ts | 103 ++++++++++++++++++ 17 files changed, 270 insertions(+), 18 deletions(-) create mode 100644 src/lib/server/autoscaling/admission.ts create mode 100644 src/lib/server/autoscaling/ownership.ts create mode 100644 tests/unit/autoscaling-live.test.ts diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index 84d351c..1b5b1ea 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -77,7 +77,7 @@ export interface ProvisioningDeployment { expected_map?: string | null; } export interface ProvisioningSource { - kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; + kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog' | 'autoscaling'; source_id: string; server?: string; label: string; diff --git a/src/lib/server/autoscaling/admission.ts b/src/lib/server/autoscaling/admission.ts new file mode 100644 index 0000000..5c089eb --- /dev/null +++ b/src/lib/server/autoscaling/admission.ts @@ -0,0 +1,32 @@ +import { db } from '../db/index.js'; +import { planScaling } from './planner.js'; +import { scalingSnapshot } from './snapshot.js'; +import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js'; +import { deploymentStage } from '../provisioning/history.js'; + +export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string } +export function scalingIntent(operationId: string): ScalingIntent | undefined { + return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined; +} +/** Rechecked before placement, after asynchronous preparation, and before dispatch. */ +export function scalingAdmission(operationId: string): string | null { + const intent = scalingIntent(operationId); + if (!intent) return null; + const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined; + let reason: string | null = null; + const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1'; + if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.'; + else { + const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' }; + const requested = JSON.parse(intent.decision) as ScalingDecision; + const snapshot = scalingSnapshot(policy, Date.now(), operationId); + if (requested.action === 'create') { + snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server); + snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`); + } + const current = planScaling(policy, snapshot, Date.now()); + if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`; + } + if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } }); + return reason; +} diff --git a/src/lib/server/autoscaling/index.ts b/src/lib/server/autoscaling/index.ts index 0f6dfda..27ff008 100644 --- a/src/lib/server/autoscaling/index.ts +++ b/src/lib/server/autoscaling/index.ts @@ -3,6 +3,9 @@ import { db } from '../db/index.js'; import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js'; import { emptyObservation, planScaling } from './planner.js'; import { scalingSnapshot } from './snapshot.js'; +import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js'; +import { cancelCreation } from '../provisioning/cancel-creation.js'; +import { scalingAdmission } from './admission.js'; export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } } interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string } @@ -19,7 +22,15 @@ export function previewScaling(input: unknown, id?: string, revision?: number): const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0, nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' }; const snapshot = scalingSnapshot(policy); - const blockers = liveScaling(policy.mode) ? ['Live autoscaling is not available yet.'] : []; + const blockers: string[] = []; + if (policy.mode === 'create-destroy') blockers.push('Automatic removal is not available yet.'); + if (liveScaling(policy.mode)) { + if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`); + blockers.push(...snapshot.blockers); + for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`); + if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.'); + try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); } + } const decision = planScaling(policy, snapshot, Date.now()); const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(), members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex'); @@ -37,21 +48,56 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at); // An edit starts a new observation interval; earlier inactivity is not authorization. db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId); + db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId); + if (liveScaling(policy.mode)) for (const member of preview.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)') + .run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() })); + reconcileScaling(getScalingPolicy(nextId)!); return getScalingPolicy(nextId)!; })(); } export function setScalingGlobalPause(paused: boolean): void { - db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0'); + db().transaction(() => { + db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0'); + if (paused) for (const policy of listScalingPolicies()) reconcileScaling(policy); + })(); } -export async function runAutoscaling(): Promise { +function reconcileScaling(policy: ScalingPolicy): void { + const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id + FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id + LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as + { id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[]; + for (const intent of intents) { + if (intent.action === 'create' && !intent.command_id && !intent.submitted_at + && (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false); + const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null }; + const terminal = ['done','warning','failed','cancelled'].includes(current.state); + const command = intent.command_id ? db().prepare('SELECT state FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string } | undefined : undefined; + const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed' + && !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id); + const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending; + if ((current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) { + db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id); + } + if (terminal && settled) db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id); + } + if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) { + db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id); + } +} + +export async function runAutoscaling(): Promise { const now = Date.now(); - for (const policy of listScalingPolicies()) { + let submitted = false; + for (const initial of listScalingPolicies()) { + reconcileScaling(initial); + const policy = getScalingPolicy(initial.id)!; if (policy.mode === 'off') continue; db().transaction(() => { const snapshot = scalingSnapshot(policy, now); if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.'); + if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.'); const decision = planScaling(policy, snapshot, now); for (const member of snapshot.members) { const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now); @@ -62,13 +108,34 @@ export async function runAutoscaling(): Promise { const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers }); const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex'); const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined; - if (last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); + const execute = liveScaling(policy.mode) && decision.action === 'create'; + const decisionId = randomUUID(); + if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)') - .run(randomUUID(), policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence); + .run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence); + if (execute) { + db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId); + try { + db().transaction(() => { + const selection = nativeLaunchSelection({ automatic: true }, policy.game); + const result = queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug, selection, + createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId }); + if (!result.ok) throw new Error(result.message); + db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId); + db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server); + db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id); + submitted = true; + })(); + } catch (error) { + db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId); + db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Creation could not be queued: ${(error as Error).message}`, policy.id); + } + } db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`) .run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty)); })(); } + return submitted; } export function pruneAutoscaling(now = Date.now()): void { @@ -93,5 +160,5 @@ export function scalingReport(): ScalingReport { COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals']; return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals }; }); - return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: false, removalAvailable: false }; + return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: false }; } diff --git a/src/lib/server/autoscaling/ownership.ts b/src/lib/server/autoscaling/ownership.ts new file mode 100644 index 0000000..5fb658f --- /dev/null +++ b/src/lib/server/autoscaling/ownership.ts @@ -0,0 +1,25 @@ +import { db } from '../db/index.js'; +import { getServer } from '../fleet/servers.js'; +import { mapIdentity } from './snapshot.js'; + +export function scalingOwner(game: string, mapSlug: string): string | null { + return (db().prepare('SELECT id FROM autoscaling_policies WHERE game=? AND map_slug=? AND owns_map=1').get(game, mapSlug) as { id: string } | undefined)?.id ?? null; +} +export function scalingOwnershipBlocker(server: string): string | null { + const identity = mapIdentity(getServer(server) ?? { name: server }); + const owner = identity && scalingOwner(identity.game, identity.mapSlug); + return owner ? `Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}` : null; +} +export function assertScalingOwnership(game: string, mapSlug: string, decisionId?: string): void { + const owner = scalingOwner(game, mapSlug); + if (!owner) { if (decisionId) throw new Error('Autoscaling no longer owns this map.'); return; } + if (decisionId && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return; + throw new Error(`Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}`); +} +/** Operator cancellation is a policy pause, not a fresh scaling signal. */ +export function pauseScalingForOperation(operationId: string): void { + const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined; + if (!row) return; + db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id); + db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id); +} diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 003d009..2a25a83 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -276,7 +276,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void { for (const name of ['provisioning_sources', 'deployment_servers']) { const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string }; const replacement = name === 'provisioning_sources' - ? (sql.includes("'watchdog'") ? sql : sql.replace("'campaign','beta','maintenance'", "'campaign','beta','maintenance','watchdog'")) + ? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'")) : sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL'); if (sql === replacement) continue; const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL") diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 63f0084..33443d4 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -527,7 +527,7 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs ( -- history retention, while current pointers live on the caller's server rows. CREATE TABLE IF NOT EXISTS provisioning_sources ( operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE, - kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog')), + kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog','autoscaling')), source_id TEXT NOT NULL, server TEXT NOT NULL, PRIMARY KEY (operation_id, kind, source_id, server) diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 26da11c..484d190 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -1,3 +1,4 @@ +import { scalingOwnershipBlocker } from '../autoscaling/ownership.js'; import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js'; @@ -97,6 +98,7 @@ function allocate(entries: BlueprintEntry[], counts = coverage(entries)): Collec function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] { const blockers: string[] = []; + for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); } if (!servers.length) return blockers; if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.'); if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); @@ -113,6 +115,8 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] { }; if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.'); for (const server of removals) { + const owned = scalingOwnershipBlocker(server.name); + if (owned) blockers.push(owned); const blocker = activeBetaRemovalBlocker(server.name); if (blocker) blockers.push(blocker); } diff --git a/src/lib/server/provisioning/cancel-creation.ts b/src/lib/server/provisioning/cancel-creation.ts index 239b5e6..685b835 100644 --- a/src/lib/server/provisioning/cancel-creation.ts +++ b/src/lib/server/provisioning/cancel-creation.ts @@ -1,3 +1,4 @@ +import { pauseScalingForOperation } from '../autoscaling/ownership.js'; import { db } from '../db/index.js'; import { nativeInstance } from '../backends/native.js'; import { queueHostOperation, releaseHostPorts } from '../hosts/index.js'; @@ -21,11 +22,12 @@ export function canCancelCreation(id: string): boolean { } /** Stops observation immediately; remote cleanup remains a durable, separately visible job. */ -export function cancelCreation(id: string, triggeredBy = 'ui'): string | null { +export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = true): string | null { return db().transaction(() => { const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined; if (prior) return prior.cleanup_id; if (!canCancelCreation(id)) return null; + if (pausePolicy) pauseScalingForOperation(id); const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as { server: string; operation_id: string | null; stage_operation_id: string | null }; diff --git a/src/lib/server/provisioning/cancellation.ts b/src/lib/server/provisioning/cancellation.ts index ed1b18a..328b13a 100644 --- a/src/lib/server/provisioning/cancellation.ts +++ b/src/lib/server/provisioning/cancellation.ts @@ -1,3 +1,4 @@ +import { pauseScalingForOperation } from '../autoscaling/ownership.js'; import { db } from '../db/index.js'; import { ensureHostSchema } from '../hosts/schema.js'; import { recordProgress } from './progress.js'; @@ -21,6 +22,7 @@ export function cancelProvisioningJob(id: string, reason = 'Cancelled by operato return db().transaction(() => { if (provisioningCancelled(id)) return true; if (!canCancelProvisioningJob(id)) return false; + pauseScalingForOperation(id); const at = new Date().toISOString(); // Only commands never delivered can be withdrawn. Delivered commands retain // their immutable IDs and receipts; replacements wait for their resolution. diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index d9a3894..8c6f510 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -1,3 +1,4 @@ +import { pauseScalingForOperation } from '../autoscaling/ownership.js'; import { randomUUID } from 'node:crypto'; import { db } from '../db/index.js'; import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js'; @@ -59,6 +60,7 @@ function warningMessages(raw: string): string[] { function sourceDetails(source: Omit): ProvisioningSource { const id = source.source_id; + if (source.kind === 'autoscaling') return { ...source, label: 'Autoscaling policy', href: `/autoscaling#${encodeURIComponent(id)}` }; if (source.kind === 'campaign') { const row = db().prepare(`SELECT d.action, d.beta_id, s.name FROM deployments d LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1)) @@ -152,6 +154,7 @@ export function cancelRemoval(id: string): boolean { const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER); if (!result.changes) return false; + pauseScalingForOperation(id); recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: 'Queued removal cancelled. The server and its files were kept.', details: { state: 'cancelled', finishedAt } } }); @@ -306,7 +309,7 @@ function deploymentListRows(query: ReturnType, activ const cache = new Map(); for (const row of rows) { const sources = byOperation.get(row.id) ?? []; - const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog):(.+)$/); + const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/); if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] }); row.sources = sources.map(source => { const key = `${source.kind}:${source.source_id}`; diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index acc92a7..3db0429 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -1,3 +1,4 @@ +import { scalingOwnershipBlocker } from '../autoscaling/ownership.js'; import { db } from '../db/index.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import type { GameMode, ZombieGame } from '../fleet/map-aliases.js'; @@ -68,6 +69,8 @@ function hasActiveCreation(name: string): boolean { } export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } { + const owned = scalingOwnershipBlocker(name); + if (owned) return { ok: false, kind: 'conflict', message: owned, warnings: [] }; const cancelled = cancelQueuedNativeCreation(name, triggeredBy); if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled }; const blocker = activeBetaRemovalBlocker(name); diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 13e8552..36bbc02 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -1,3 +1,4 @@ +import { assertScalingOwnership } from '../autoscaling/ownership.js'; import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js'; import { concurrencyBlocker, preparingJobs } from './concurrency.js'; import { isIP } from 'node:net'; @@ -152,11 +153,14 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul /** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */ export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection; - plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan }): CreateServerResult { + plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan; autoscalingDecisionId?: string }): CreateServerResult { try { return db().transaction(() => { + assertScalingOwnership(input.game, input.mapSlug, input.autoscalingDecisionId); const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`; - if (input.server !== expected || nativeInstance(input.server)) throw new Error('Available lobby changed. Review a fresh preview.'); + const reserved = db().prepare(`SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=? + AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))`).get(input.server); + if ((!input.autoscalingDecisionId && input.server !== expected) || nativeInstance(input.server) || getServer(input.server) || reserved) throw new Error('Available lobby changed. Review a fresh preview.'); const key = input.plutoKey?.trim() || getServerKey(input.server); if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`); saveServerKey(input.server, key); @@ -402,7 +406,7 @@ function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boo } /** Capacity history and host ranking run in the background reader, one job at a time. */ -async function placeQueuedCreation(job: Job, instance: NativeInstance | null): Promise { +async function placeQueuedCreation(job: Job, instance: NativeInstance | null, eligibility: (job: Job) => string | null): Promise { const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe; const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!; const network = prior.automatic ? prior.network ?? {} : prior; @@ -413,6 +417,7 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P return db().transaction(() => { const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; if (state?.state !== 'queued') return null; + if (eligibility(job)) return null; const live = nativeInstance(job.server); if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null; const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL @@ -467,7 +472,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const capacityBlocker = concurrencyBlocker(job.id, job.action); if (capacityBlocker) { hold(job, capacityBlocker); return; } if (eligibility(job)) return; - instance = await placeQueuedCreation(job, instance); + instance = await placeQueuedCreation(job, instance, eligibility); if (!instance) return; job.instance_id = instance.id; job.generation = instance.generation; } diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 4dd4a47..48160d6 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -1,3 +1,4 @@ +import { scalingOwnershipBlocker } from '../autoscaling/ownership.js'; import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; @@ -120,6 +121,8 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; + const owned = action !== 'restart' ? scalingOwnershipBlocker(server) : null; + if (owned) return { ok: false as const, status: 409, error: owned }; if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' }; return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index 370a02b..0d96024 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -1,3 +1,4 @@ +import { scalingAdmission } from '../autoscaling/admission.js'; import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { getServer } from '../fleet/servers.js'; @@ -93,6 +94,8 @@ export const processLifecycle = processProvisioning; function nativePass(options: { wait?: boolean }): Promise { return processNativeProvisioning(operation => { + const scaling = scalingAdmission(operation.id); + if (scaling) return scaling; if (operation.action === 'create') return null; if (operation.action === 'destroy') { const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL diff --git a/src/lib/ui/provisioning-presentation.ts b/src/lib/ui/provisioning-presentation.ts index cbc6fa1..c471255 100644 --- a/src/lib/ui/provisioning-presentation.ts +++ b/src/lib/ui/provisioning-presentation.ts @@ -179,7 +179,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi if (!record(value.activity) || !number(value.activity.total) || !(value.activity.latestId === null || number(value.activity.latestId)) || !(value.activity.failureExcerpt === null || event(value.activity.failureExcerpt))) return false; if (!Array.isArray(value.sources) || !value.sources.every(source => record(source) - && ['campaign', 'beta', 'maintenance', 'watchdog'].includes(source.kind as string) + && ['campaign', 'beta', 'maintenance', 'watchdog', 'autoscaling'].includes(source.kind as string) && text(source.source_id) && text(source.label) && localLink(source.href))) return false; if (!(value.collection === null || record(value.collection) && text(value.collection.id) && text(value.collection.name))) return false; if (!record(value.capacity) || !timestamp(value.capacity.evaluatedAt) || !Array.isArray(value.capacity.pools) diff --git a/tests/unit/autoscaling-dry-run.test.ts b/tests/unit/autoscaling-dry-run.test.ts index 05daaac..b36c16c 100644 --- a/tests/unit/autoscaling-dry-run.test.ts +++ b/tests/unit/autoscaling-dry-run.test.ts @@ -31,7 +31,7 @@ it('rejects duplicate policies, stale edits and changed previews', async () => { expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists'); expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed'); expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed'); - expect(() => scaling.saveScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision).token)).toThrow('not available'); + expect(() => scaling.saveScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision).token)).toThrow('not available'); }); it('pauses evaluations and prunes expired observations without deleting current history', async () => { diff --git a/tests/unit/autoscaling-live.test.ts b/tests/unit/autoscaling-live.test.ts new file mode 100644 index 0000000..1549dae --- /dev/null +++ b/tests/unit/autoscaling-live.test.ts @@ -0,0 +1,103 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { prepareNativeHost } from '../helpers/native-host.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { await fixture.cleanup(); }); +const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-only' }; +async function setup() { + await prepareNativeHost(); + const { db } = await import('../../src/lib/server/db/index.js'); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + for (const n of [3, 8]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`); + const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + return { db, scaling, saved }; +} + +it('admits one durable keyed create and reuses it across ticks and restart', async () => { + const { db, scaling, saved } = await setup(); + expect(await scaling.runAutoscaling()).toBe(true); + await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]); + const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); + await scaling.runAutoscaling(); + expect(db().prepare('SELECT server,state FROM server_deployments WHERE action=\'create\'').all()).toEqual([{ server: 't6_zombie_town_3', state: 'queued' }]); + expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 1 }); + const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string }; + const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); + expect(scalingAdmission(id)).toBeNull(); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + expect(getDeployment(id)?.sources).toContainEqual(expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` })); +}); + +it('requires key coverage and withdraws an unsubmitted create when the policy pauses', async () => { + const { db, scaling, saved } = await setup(); + await scaling.runAutoscaling(); + const off = { ...settings, mode: 'off' }; + scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token); + expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' }); + expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 0 }); + db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run(); + const current = scaling.getScalingPolicy(saved.id)!; + expect(scaling.previewScaling(settings, saved.id, current.revision).blockers.join(' ')).toContain('Save keys'); +}); + +it('blocks manual lifecycle and collection creation, including stale previews', async () => { + const { db, scaling, saved } = await setup(); + const { queueCreateServer, destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); + expect(destroyServer('t6_zombie_town_3')).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); + expect(requestLifecycle('t6_zombie_town_3', 'stop')).toMatchObject({ ok: false, status: 409 }); + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ name: 'Town', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] }); + expect(previewCollection(blueprint.id).blockers.join(' ')).toContain('Autoscaling'); + const off = { ...settings, mode: 'off' }; scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token); + const preview = previewCollection(blueprint.id); + const current = scaling.getScalingPolicy(saved.id)!; + scaling.saveScaling(settings, saved.id, current.revision, scaling.previewScaling(settings, saved.id, current.revision).token); + expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling'); + expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 }); +}); + +it('operator cancellation pauses the policy and does not immediately create a replacement', async () => { + const { db, scaling } = await setup(); + await scaling.runAutoscaling(); + const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string }; + const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); + expect(cancelProvisioningJob(id)).toBe(true); + await scaling.runAutoscaling(); + expect(scaling.scalingReport().policies[0].policy.mode).toBe('off'); + expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()).toEqual({ n: 1 }); +}); + +it('exact matching cannot remove an owned map absent from the collection', async () => { + const town = 't6_zombie_town_1'; + await prepareNativeHost([town]); + const { db } = await import('../../src/lib/server/db/index.js'); + const at = new Date().toISOString(); + db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at); + for (const n of [1, 2]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const collection = saveBlueprint({ name: 'Moon only', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] }); + const preview = previewCollection(collection.id, 'exact'); + expect(preview.removals.map(r => r.name)).toContain(town); + expect(preview.blockers.join(' ')).toContain('Autoscaling'); + expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling'); +}); + +it('rechecks evidence and global pause before dispatch', async () => { + const { db, scaling } = await setup(); + await scaling.runAutoscaling(); + const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string }; + const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); + db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_3'").run(); + expect(scalingAdmission(id)).toContain('no longer eligible'); + scaling.setScalingGlobalPause(true); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' }); + expect(await scaling.runAutoscaling()).toBe(false); +});