josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add exclusive map ownership and capacity-aware autoscaling creation

Browse Source
This commit is contained in:
josh
2026-10-06 22:59:17 -04:00
parent 36cf2b9c05
commit e012278bd9
17 changed files with 270 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/provisioning-detail.ts
+1 -1
View File
@@ -77,7 +77,7 @@ export interface ProvisioningDeployment {
expected_map?: string | null;
}
export interface ProvisioningSource {
kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog';
kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog' | 'autoscaling';
source_id: string;
server?: string;
label: string;
src/lib/server/autoscaling/admission.ts
+32
View File
@@ -0,0 +1,32 @@
import { db } from '../db/index.js';
import { planScaling } from './planner.js';
import { scalingSnapshot } from './snapshot.js';
import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
import { deploymentStage } from '../provisioning/history.js';
export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
export function scalingIntent(operationId: string): ScalingIntent | undefined {
return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
}
/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
export function scalingAdmission(operationId: string): string | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
let reason: string | null = null;
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
}
const current = planScaling(policy, snapshot, Date.now());
if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
}
if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
return reason;
}
src/lib/server/autoscaling/index.ts
+74 -7
View File
@@ -3,6 +3,9 @@ import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingPreview, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { scalingSnapshot } from './snapshot.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { cancelCreation } from '../provisioning/cancel-creation.js';
import { scalingAdmission } from './admission.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; next_action_at: number; paused_reason: string | null; updated_at: string }
@@ -19,7 +22,15 @@ export function previewScaling(input: unknown, id?: string, revision?: number):
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'preview', revision: existing?.revision ?? 0,
nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
const snapshot = scalingSnapshot(policy);
const blockers = liveScaling(policy.mode) ? ['Live autoscaling is not available yet.'] : [];
const blockers: string[] = [];
if (policy.mode === 'create-destroy') blockers.push('Automatic removal is not available yet.');
if (liveScaling(policy.mode)) {
if (snapshot.keyedNames.length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities before activating this policy.`);
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
}
const decision = planScaling(policy, snapshot, Date.now());
const token = createHash('sha256').update(JSON.stringify({ settings, revision: policy.revision, globalPaused: scalingGloballyPaused(),
members: snapshot.members.map(m => [m.name, m.identity, m.state]), keys: snapshot.keyedNames.sort(), reservations: snapshot.reservedNames.sort(), blockers: snapshot.blockers, decision })).digest('hex');
@@ -37,21 +48,56 @@ export function saveScaling(input: unknown, id?: string, revision?: number, toke
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode)) for (const member of preview.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
.run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
reconcileScaling(getScalingPolicy(nextId)!);
return getScalingPolicy(nextId)!;
})();
}
export function setScalingGlobalPause(paused: boolean): void {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
db().transaction(() => {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
if (paused) for (const policy of listScalingPolicies()) reconcileScaling(policy);
})();
}
export async function runAutoscaling(): Promise<void> {
function reconcileScaling(policy: ScalingPolicy): void {
const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
{ id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
for (const intent of intents) {
if (intent.action === 'create' && !intent.command_id && !intent.submitted_at
&& (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done','warning','failed','cancelled'].includes(current.state);
const command = intent.command_id ? db().prepare('SELECT state FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string } | undefined : undefined;
const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
&& !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
if ((current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
}
if (terminal && settled) db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
}
if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
}
}
export async function runAutoscaling(): Promise<boolean> {
const now = Date.now();
for (const policy of listScalingPolicies()) {
let submitted = false;
for (const initial of listScalingPolicies()) {
reconcileScaling(initial);
const policy = getScalingPolicy(initial.id)!;
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.');
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
@@ -62,13 +108,34 @@ export async function runAutoscaling(): Promise<void> {
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyCount: snapshot.keyedNames.length, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
if (last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
const execute = liveScaling(policy.mode) && decision.action === 'create';
const decisionId = randomUUID();
if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(randomUUID(), policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
.run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
if (execute) {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
try {
db().transaction(() => {
const selection = nativeLaunchSelection({ automatic: true }, policy.game);
const result = queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug, selection,
createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId });
if (!result.ok) throw new Error(result.message);
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Creation could not be queued: ${(error as Error).message}`, policy.id);
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, Math.max(0, policy.spares - decision.empty));
})();
}
return submitted;
}
export function pruneAutoscaling(now = Date.now()): void {
@@ -93,5 +160,5 @@ export function scalingReport(): ScalingReport {
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: false, removalAvailable: false };
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: false };
}
src/lib/server/autoscaling/ownership.ts
+25
View File
@@ -0,0 +1,25 @@
import { db } from '../db/index.js';
import { getServer } from '../fleet/servers.js';
import { mapIdentity } from './snapshot.js';
export function scalingOwner(game: string, mapSlug: string): string | null {
return (db().prepare('SELECT id FROM autoscaling_policies WHERE game=? AND map_slug=? AND owns_map=1').get(game, mapSlug) as { id: string } | undefined)?.id ?? null;
}
export function scalingOwnershipBlocker(server: string): string | null {
const identity = mapIdentity(getServer(server) ?? { name: server });
const owner = identity && scalingOwner(identity.game, identity.mapSlug);
return owner ? `Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}` : null;
}
export function assertScalingOwnership(game: string, mapSlug: string, decisionId?: string): void {
const owner = scalingOwner(game, mapSlug);
if (!owner) { if (decisionId) throw new Error('Autoscaling no longer owns this map.'); return; }
if (decisionId && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
throw new Error(`Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}`);
}
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
export function pauseScalingForOperation(operationId: string): void {
const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
if (!row) return;
db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
}
src/lib/server/db/index.ts
+1 -1
View File
@@ -276,7 +276,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void {
for (const name of ['provisioning_sources', 'deployment_servers']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = name === 'provisioning_sources'
? (sql.includes("'watchdog'") ? sql : sql.replace("'campaign','beta','maintenance'", "'campaign','beta','maintenance','watchdog'"))
? (sql.includes("'autoscaling'") ? sql : sql.replace(/'campaign','beta','maintenance'(?:,'watchdog')?/, "'campaign','beta','maintenance','watchdog','autoscaling'"))
: sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
if (sql === replacement) continue;
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
src/lib/server/db/schema.sql
+1 -1
View File
@@ -527,7 +527,7 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
-- history retention, while current pointers live on the caller's server rows.
CREATE TABLE IF NOT EXISTS provisioning_sources (
operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog')),
kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance','watchdog','autoscaling')),
source_id TEXT NOT NULL,
server TEXT NOT NULL,
PRIMARY KEY (operation_id, kind, source_id, server)
src/lib/server/fleet/collections.ts
+4
View File
@@ -1,3 +1,4 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
@@ -97,6 +98,7 @@ function allocate(entries: BlueprintEntry[], counts = coverage(entries)): Collec
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
const blockers: string[] = [];
for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); }
if (!servers.length) return blockers;
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
@@ -113,6 +115,8 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
};
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const owned = scalingOwnershipBlocker(server.name);
if (owned) blockers.push(owned);
const blocker = activeBetaRemovalBlocker(server.name);
if (blocker) blockers.push(blocker);
}
src/lib/server/provisioning/cancel-creation.ts
+3 -1
View File
@@ -1,3 +1,4 @@
import { pauseScalingForOperation } from '../autoscaling/ownership.js';
import { db } from '../db/index.js';
import { nativeInstance } from '../backends/native.js';
import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
@@ -21,11 +22,12 @@ export function canCancelCreation(id: string): boolean {
}
/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
export function cancelCreation(id: string, triggeredBy = 'ui', pausePolicy = true): string | null {
return db().transaction(() => {
const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
if (prior) return prior.cleanup_id;
if (!canCancelCreation(id)) return null;
if (pausePolicy) pauseScalingForOperation(id);
const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
{ server: string; operation_id: string | null; stage_operation_id: string | null };
src/lib/server/provisioning/cancellation.ts
+2
View File
@@ -1,3 +1,4 @@
import { pauseScalingForOperation } from '../autoscaling/ownership.js';
import { db } from '../db/index.js';
import { ensureHostSchema } from '../hosts/schema.js';
import { recordProgress } from './progress.js';
@@ -21,6 +22,7 @@ export function cancelProvisioningJob(id: string, reason = 'Cancelled by operato
return db().transaction(() => {
if (provisioningCancelled(id)) return true;
if (!canCancelProvisioningJob(id)) return false;
pauseScalingForOperation(id);
const at = new Date().toISOString();
// Only commands never delivered can be withdrawn. Delivered commands retain
// their immutable IDs and receipts; replacements wait for their resolution.
src/lib/server/provisioning/history.ts
+4 -1
View File
@@ -1,3 +1,4 @@
import { pauseScalingForOperation } from '../autoscaling/ownership.js';
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
@@ -59,6 +60,7 @@ function warningMessages(raw: string): string[] {
function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
const id = source.source_id;
if (source.kind === 'autoscaling') return { ...source, label: 'Autoscaling policy', href: `/autoscaling#${encodeURIComponent(id)}` };
if (source.kind === 'campaign') {
const row = db().prepare(`SELECT d.action, d.beta_id, s.name FROM deployments d
LEFT JOIN scripts s ON s.id=COALESCE(d.script_id, (SELECT script_id FROM deployment_scripts WHERE deployment_id=d.id ORDER BY script_id LIMIT 1))
@@ -152,6 +154,7 @@ export function cancelRemoval(id: string): boolean {
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
if (!result.changes) return false;
pauseScalingForOperation(id);
recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
message: 'Queued removal cancelled. The server and its files were kept.',
details: { state: 'cancelled', finishedAt } } });
@@ -306,7 +309,7 @@ function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, activ
const cache = new Map<string, ProvisioningSource>();
for (const row of rows) {
const sources = byOperation.get(row.id) ?? [];
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog):(.+)$/);
const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/);
if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
row.sources = sources.map(source => {
const key = `${source.kind}:${source.source_id}`;
src/lib/server/provisioning/instances.ts
+3
View File
@@ -1,3 +1,4 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { db } from '../db/index.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import type { GameMode, ZombieGame } from '../fleet/map-aliases.js';
@@ -68,6 +69,8 @@ function hasActiveCreation(name: string): boolean {
}
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
const owned = scalingOwnershipBlocker(name);
if (owned) return { ok: false, kind: 'conflict', message: owned, warnings: [] };
const cancelled = cancelQueuedNativeCreation(name, triggeredBy);
if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled };
const blocker = activeBetaRemovalBlocker(name);
src/lib/server/provisioning/native.ts
+9 -4
View File
@@ -1,3 +1,4 @@
import { assertScalingOwnership } from '../autoscaling/ownership.js';
import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
import { concurrencyBlocker, preparingJobs } from './concurrency.js';
import { isIP } from 'node:net';
@@ -152,11 +153,14 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
/** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */
export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection;
plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan }): CreateServerResult {
plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan; autoscalingDecisionId?: string }): CreateServerResult {
try {
return db().transaction(() => {
assertScalingOwnership(input.game, input.mapSlug, input.autoscalingDecisionId);
const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`;
if (input.server !== expected || nativeInstance(input.server)) throw new Error('Available lobby changed. Review a fresh preview.');
const reserved = db().prepare(`SELECT 1 FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE j.server=?
AND d.server_deleted_at IS NULL AND (d.state IN ('queued','waiting','running','awaiting-health','interrupted') OR (d.state='cancelled' AND d.submitted_at IS NOT NULL))`).get(input.server);
if ((!input.autoscalingDecisionId && input.server !== expected) || nativeInstance(input.server) || getServer(input.server) || reserved) throw new Error('Available lobby changed. Review a fresh preview.');
const key = input.plutoKey?.trim() || getServerKey(input.server);
if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`);
saveServerKey(input.server, key);
@@ -402,7 +406,7 @@ function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boo
}
/** Capacity history and host ranking run in the background reader, one job at a time. */
async function placeQueuedCreation(job: Job, instance: NativeInstance | null): Promise<NativeInstance | null> {
async function placeQueuedCreation(job: Job, instance: NativeInstance | null, eligibility: (job: Job) => string | null): Promise<NativeInstance | null> {
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!;
const network = prior.automatic ? prior.network ?? {} : prior;
@@ -413,6 +417,7 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
return db().transaction(() => {
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (state?.state !== 'queued') return null;
if (eligibility(job)) return null;
const live = nativeInstance(job.server);
if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null;
const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
@@ -467,7 +472,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const capacityBlocker = concurrencyBlocker(job.id, job.action);
if (capacityBlocker) { hold(job, capacityBlocker); return; }
if (eligibility(job)) return;
instance = await placeQueuedCreation(job, instance);
instance = await placeQueuedCreation(job, instance, eligibility);
if (!instance) return;
job.instance_id = instance.id; job.generation = instance.generation;
}
src/lib/server/provisioning/requests.ts
+3
View File
@@ -1,3 +1,4 @@
import { scalingOwnershipBlocker } from '../autoscaling/ownership.js';
import { maintenanceAllowed } from '../maintenance/policies.js';
import { db } from '../db/index.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -120,6 +121,8 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
export function requestLifecycle(server: string, action: RuntimeAction) {
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
const owned = action !== 'restart' ? scalingOwnershipBlocker(server) : null;
if (owned) return { ok: false as const, status: 409, error: owned };
if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
return db().transaction(() => {
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
src/lib/server/provisioning/worker.ts
+3
View File
@@ -1,3 +1,4 @@
import { scalingAdmission } from '../autoscaling/admission.js';
import { maintenanceAllowed } from '../maintenance/policies.js';
import { db } from '../db/index.js';
import { getServer } from '../fleet/servers.js';
@@ -93,6 +94,8 @@ export const processLifecycle = processProvisioning;
function nativePass(options: { wait?: boolean }): Promise<void> {
return processNativeProvisioning(operation => {
const scaling = scalingAdmission(operation.id);
if (scaling) return scaling;
if (operation.action === 'create') return null;
if (operation.action === 'destroy') {
const other = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND id!=? AND server_deleted_at IS NULL
src/lib/ui/provisioning-presentation.ts
+1 -1
View File
@@ -179,7 +179,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
if (!record(value.activity) || !number(value.activity.total) || !(value.activity.latestId === null || number(value.activity.latestId))
|| !(value.activity.failureExcerpt === null || event(value.activity.failureExcerpt))) return false;
if (!Array.isArray(value.sources) || !value.sources.every(source => record(source)
&& ['campaign', 'beta', 'maintenance', 'watchdog'].includes(source.kind as string)
&& ['campaign', 'beta', 'maintenance', 'watchdog', 'autoscaling'].includes(source.kind as string)
&& text(source.source_id) && text(source.label) && localLink(source.href))) return false;
if (!(value.collection === null || record(value.collection) && text(value.collection.id) && text(value.collection.name))) return false;
if (!record(value.capacity) || !timestamp(value.capacity.evaluatedAt) || !Array.isArray(value.capacity.pools)
tests/unit/autoscaling-dry-run.test.ts
+1 -1
View File
@@ -31,7 +31,7 @@ it('rejects duplicate policies, stale edits and changed previews', async () => {
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists');
expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
expect(() => scaling.saveScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-only' }, saved.id, saved.revision).token)).toThrow('not available');
expect(() => scaling.saveScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision).token)).toThrow('not available');
});
it('pauses evaluations and prunes expired observations without deleting current history', async () => {
tests/unit/autoscaling-live.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { prepareNativeHost } from '../helpers/native-host.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-only' };
async function setup() {
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
for (const n of [3, 8]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
return { db, scaling, saved };
}
it('admits one durable keyed create and reuses it across ticks and restart', async () => {
const { db, scaling, saved } = await setup();
expect(await scaling.runAutoscaling()).toBe(true);
await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
await scaling.runAutoscaling();
expect(db().prepare('SELECT server,state FROM server_deployments WHERE action=\'create\'').all()).toEqual([{ server: 't6_zombie_town_3', state: 'queued' }]);
expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 1 });
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
expect(scalingAdmission(id)).toBeNull();
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
expect(getDeployment(id)?.sources).toContainEqual(expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` }));
});
it('requires key coverage and withdraws an unsubmitted create when the policy pauses', async () => {
const { db, scaling, saved } = await setup();
await scaling.runAutoscaling();
const off = { ...settings, mode: 'off' };
scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' });
expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 0 });
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run();
const current = scaling.getScalingPolicy(saved.id)!;
expect(scaling.previewScaling(settings, saved.id, current.revision).blockers.join(' ')).toContain('Save keys');
});
it('blocks manual lifecycle and collection creation, including stale previews', async () => {
const { db, scaling, saved } = await setup();
const { queueCreateServer, destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
expect(destroyServer('t6_zombie_town_3')).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
expect(requestLifecycle('t6_zombie_town_3', 'stop')).toMatchObject({ ok: false, status: 409 });
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Town', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] });
expect(previewCollection(blueprint.id).blockers.join(' ')).toContain('Autoscaling');
const off = { ...settings, mode: 'off' }; scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
const preview = previewCollection(blueprint.id);
const current = scaling.getScalingPolicy(saved.id)!;
scaling.saveScaling(settings, saved.id, current.revision, scaling.previewScaling(settings, saved.id, current.revision).token);
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 });
});
it('operator cancellation pauses the policy and does not immediately create a replacement', async () => {
const { db, scaling } = await setup();
await scaling.runAutoscaling();
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
expect(cancelProvisioningJob(id)).toBe(true);
await scaling.runAutoscaling();
expect(scaling.scalingReport().policies[0].policy.mode).toBe('off');
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()).toEqual({ n: 1 });
});
it('exact matching cannot remove an owned map absent from the collection', async () => {
const town = 't6_zombie_town_1';
await prepareNativeHost([town]);
const { db } = await import('../../src/lib/server/db/index.js');
const at = new Date().toISOString();
db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at);
for (const n of [1, 2]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
const scaling = await import('../../src/lib/server/autoscaling/index.js');
scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const collection = saveBlueprint({ name: 'Moon only', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
const preview = previewCollection(collection.id, 'exact');
expect(preview.removals.map(r => r.name)).toContain(town);
expect(preview.blockers.join(' ')).toContain('Autoscaling');
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
});
it('rechecks evidence and global pause before dispatch', async () => {
const { db, scaling } = await setup();
await scaling.runAutoscaling();
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_3'").run();
expect(scalingAdmission(id)).toContain('no longer eligible');
scaling.setScalingGlobalPause(true);
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' });
expect(await scaling.runAutoscaling()).toBe(false);
});