josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/autoscaling/admission.ts
+55 -10
View File
@@ -4,29 +4,74 @@ import { scalingSnapshot } from './snapshot.js';
import { liveScaling, type ScalingPolicy, type ScalingDecision } from '../../autoscaling.js';
import { deploymentStage } from '../provisioning/history.js';
export interface ScalingIntent { policy_id: string; revision: number; decision: string; operation_id: string }
export interface ScalingIntent {
policy_id: string;
revision: number;
decision: string;
operation_id: string;
}
export function scalingIntent(operationId: string): ScalingIntent | undefined {
return db().prepare('SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0').get(operationId) as ScalingIntent | undefined;
return db()
.prepare(
'SELECT policy_id,revision,decision,operation_id FROM autoscaling_decisions WHERE operation_id=? AND resolved=0'
)
.get(operationId) as ScalingIntent | undefined;
}
/** Rechecked before placement, after asynchronous preparation, and before dispatch. */
export function scalingAdmission(operationId: string): string | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(intent.policy_id) as { id: string; settings: string; mode: ScalingPolicy['mode']; revision: number; paused_reason: string | null; owns_map: number } | undefined;
const row = db()
.prepare('SELECT * FROM autoscaling_policies WHERE id=?')
.get(intent.policy_id) as
| {
id: string;
settings: string;
mode: ScalingPolicy['mode'];
revision: number;
paused_reason: string | null;
owns_map: number;
}
| undefined;
let reason: string | null = null;
const paused = (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
if (!row || !liveScaling(row.mode) || !row.owns_map || row.revision !== intent.revision || row.paused_reason || paused) reason = 'Autoscaling policy is paused or changed.';
const paused =
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
if (
!row ||
!liveScaling(row.mode) ||
!row.owns_map ||
row.revision !== intent.revision ||
row.paused_reason ||
paused
)
reason = 'Autoscaling policy is paused or changed.';
else {
const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = {
...JSON.parse(row.settings),
id: row.id,
mode: row.mode,
revision: row.revision,
pausedReason: null,
updatedAt: ''
};
const requested = JSON.parse(intent.decision) as ScalingDecision;
const snapshot = scalingSnapshot(policy, Date.now(), operationId);
if (requested.action === 'create') {
snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(b => b !== `Unresolved native instance ${requested.server}.`);
snapshot.reservedNames = snapshot.reservedNames.filter((n) => n !== requested.server);
snapshot.blockers = snapshot.blockers.filter(
(b) => b !== `Unresolved native instance ${requested.server}.`
);
}
const current = planScaling(policy, snapshot, Date.now());
if (current.action !== requested.action || current.server !== requested.server) reason = `Autoscaling intent no longer eligible: ${current.reason}`;
if (current.action !== requested.action || current.server !== requested.server)
reason = `Autoscaling intent no longer eligible: ${current.reason}`;
}
if (reason) deploymentStage(operationId, reason, { blocker: { code: 'autoscaling', message: reason, automatic: true } });
if (reason)
deploymentStage(operationId, reason, {
blocker: { code: 'autoscaling', message: reason, automatic: true }
});
return reason;
}
src/lib/server/autoscaling/authorization.ts
+77 -17
View File
@@ -7,46 +7,106 @@ export const autoscalingRemovalCapability = 'instance.autoscaling-remove';
export function removalAuthorization(operationId: string, hostId: string): object | null {
const intent = scalingIntent(operationId);
if (!intent) return null;
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) throw new Error('Update the host agent before automatic removal.');
if (!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability))
throw new Error('Update the host agent before automatic removal.');
const reason = scalingAdmission(operationId);
if (reason) throw new Error(reason);
const expiresAt = Date.now() + 60_000;
db().prepare('INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)')
db()
.prepare(
'INSERT INTO autoscaling_authorizations(operation_id,policy_id,revision,expires_at) VALUES(?,?,?,?)'
)
.run(operationId, intent.policy_id, intent.revision, expiresAt);
return { policyId: intent.policy_id, revision: intent.revision, expiresAt };
}
/** Claim is the cancellation boundary. Lost replies never authorize a second claim. */
export function claimRemovalAuthorization(hostId: string, commandId: string, input: Record<string, unknown>): { authorized: boolean; validForMs?: number; reason?: string } {
export function claimRemovalAuthorization(
hostId: string,
commandId: string,
input: Record<string, unknown>
): { authorized: boolean; validForMs?: number; reason?: string } {
return db().transaction(() => {
const now = Date.now();
const row = db().prepare(`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
const row = db()
.prepare(
`SELECT a.*,d.state AS deployment_state,o.state AS command_state FROM autoscaling_authorizations a
JOIN native_provisioning n ON n.deployment_id=a.operation_id JOIN server_deployments d ON d.id=n.deployment_id
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
JOIN ec_host_operations o ON o.id=n.operation_id AND o.host_id=i.host_id AND o.instance_id=i.id AND o.generation=i.generation
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`).get(commandId, hostId) as
{ operation_id: string; policy_id: string; revision: number; expires_at: number; claimed_at: number | null; revoked_at: number | null; deployment_state: string; command_state: string } | undefined;
if (!row || input.policyId !== row.policy_id || input.revision !== row.revision || input.expiresAt !== row.expires_at
|| row.claimed_at !== null || row.revoked_at !== null || row.expires_at <= now
|| row.deployment_state !== 'running' || !['pending','accepted','running'].includes(row.command_state)) return { authorized: false, reason: 'Removal authorization expired, changed, or was already claimed.' };
WHERE o.id=? AND o.host_id=? AND d.action='destroy' AND d.server_deleted_at IS NULL`
)
.get(commandId, hostId) as
| {
operation_id: string;
policy_id: string;
revision: number;
expires_at: number;
claimed_at: number | null;
revoked_at: number | null;
deployment_state: string;
command_state: string;
}
| undefined;
if (
!row ||
input.policyId !== row.policy_id ||
input.revision !== row.revision ||
input.expiresAt !== row.expires_at ||
row.claimed_at !== null ||
row.revoked_at !== null ||
row.expires_at <= now ||
row.deployment_state !== 'running' ||
!['pending', 'accepted', 'running'].includes(row.command_state)
)
return {
authorized: false,
reason: 'Removal authorization expired, changed, or was already claimed.'
};
const reason = scalingAdmission(row.operation_id);
if (reason) { db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?').run(now, row.operation_id); return { authorized: false, reason }; }
db().prepare('UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL').run(now, row.operation_id);
if (reason) {
db()
.prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE operation_id=?')
.run(now, row.operation_id);
return { authorized: false, reason };
}
db()
.prepare(
'UPDATE autoscaling_authorizations SET claimed_at=? WHERE operation_id=? AND claimed_at IS NULL'
)
.run(now, row.operation_id);
// Only a short final occupancy check may occur after this boundary.
return { authorized: true, validForMs: Math.min(5000, row.expires_at - now) };
})();
}
export function revokePolicyRemovals(policyId: string): void {
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId)) return;
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL').run(Date.now(), policyId);
if (!db().prepare('SELECT 1 FROM autoscaling_authorizations WHERE policy_id=?').get(policyId))
return;
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL AND revoked_at IS NULL'
)
.run(Date.now(), policyId);
const at = new Date().toISOString();
const commands = db().prepare(`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
const commands = db()
.prepare(
`SELECT o.id FROM ec_host_operations o JOIN native_provisioning n ON n.operation_id=o.id
JOIN autoscaling_authorizations a ON a.operation_id=n.deployment_id
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`).all(policyId) as { id: string }[];
WHERE a.policy_id=? AND a.claimed_at IS NULL AND a.revoked_at IS NOT NULL AND o.state='pending' AND o.last_delivered_at IS NULL`
)
.all(policyId) as { id: string }[];
for (const command of commands) {
const result = JSON.stringify({ autoscalingInvalidated: true });
db().prepare("UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?").run(result, at, command.id);
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')").run(command.id, at);
db()
.prepare(
"UPDATE ec_host_operations SET state='failed',result=?,error='Autoscaling removal withdrawn before delivery.',updated_at=? WHERE id=?"
)
.run(result, at, command.id);
db()
.prepare(
"INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Autoscaling removal withdrawn before delivery.')"
)
.run(command.id, at);
}
}
src/lib/server/autoscaling/index.ts
+428 -87
View File
@@ -1,6 +1,13 @@
import { createHash, randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { liveScaling, scalingSettings, type ScalingPolicy, type ScalingEvaluation, type ScalingReport, type ScalingView } from '../../autoscaling.js';
import {
liveScaling,
scalingSettings,
type ScalingPolicy,
type ScalingEvaluation,
type ScalingReport,
type ScalingView
} from '../../autoscaling.js';
import { emptyObservation, planScaling } from './planner.js';
import { parseServerName } from '../../map-aliases.js';
import { scalingSnapshot } from './snapshot.js';
@@ -14,33 +21,98 @@ import { getHost } from '../hosts/index.js';
import { scalingOwner } from './ownership.js';
import { getServer } from '../fleet/servers.js';
export class ScalingError extends Error { constructor(message: string, public status = 400) { super(message); } }
interface PolicyRow { id: string; revision: number; settings: string; mode: ScalingPolicy['mode']; paused_reason: string | null; updated_at: string }
const present = (row: PolicyRow): ScalingPolicy => ({ ...JSON.parse(row.settings), id: row.id, revision: row.revision, mode: row.mode, pausedReason: row.paused_reason, updatedAt: row.updated_at });
export const listScalingPolicies = (): ScalingPolicy[] => (db().prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug').all() as PolicyRow[]).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined { const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as PolicyRow | undefined; return row && present(row); }
export const scalingGloballyPaused = (): boolean => (db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1';
export class ScalingError extends Error {
constructor(
message: string,
public status = 400
) {
super(message);
}
}
interface PolicyRow {
id: string;
revision: number;
settings: string;
mode: ScalingPolicy['mode'];
paused_reason: string | null;
updated_at: string;
}
const present = (row: PolicyRow): ScalingPolicy => ({
...JSON.parse(row.settings),
id: row.id,
revision: row.revision,
mode: row.mode,
pausedReason: row.paused_reason,
updatedAt: row.updated_at
});
export const listScalingPolicies = (): ScalingPolicy[] =>
(
db()
.prepare('SELECT * FROM autoscaling_policies ORDER BY game,map_slug')
.all() as PolicyRow[]
).map(present);
export function getScalingPolicy(id: string): ScalingPolicy | undefined {
const row = db().prepare('SELECT * FROM autoscaling_policies WHERE id=?').get(id) as
PolicyRow | undefined;
return row && present(row);
}
export const scalingGloballyPaused = (): boolean =>
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1';
export function evaluateScaling(input: unknown, id?: string, revision?: number): ScalingEvaluation {
const settings = scalingSettings(input), existing = id ? getScalingPolicy(id) : undefined;
const settings = scalingSettings(input),
existing = id ? getScalingPolicy(id) : undefined;
if (id && !existing) throw new ScalingError('Policy not found.', 404);
if (existing && existing.revision !== revision) throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug)) throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = { ...settings, id: existing?.id ?? 'new-policy', revision: existing?.revision ?? 0,
pausedReason: null, updatedAt: existing?.updatedAt ?? '' };
if (existing && existing.revision !== revision)
throw new ScalingError('Policy changed. Refresh before editing.', 409);
if (existing && (existing.game !== settings.game || existing.mapSlug !== settings.mapSlug))
throw new ScalingError('A policy cannot change its game or map.');
const policy: ScalingPolicy = {
...settings,
id: existing?.id ?? 'new-policy',
revision: existing?.revision ?? 0,
pausedReason: null,
updatedAt: existing?.updatedAt ?? ''
};
const snapshot = scalingSnapshot(policy);
const blockers: string[] = [];
if (liveScaling(policy.mode)) {
const owner = scalingOwner(policy.game, policy.mapSlug);
if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`);
if (owner && owner !== policy.id)
blockers.push(
`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`
);
blockers.push(...snapshot.blockers);
for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`);
if (policy.mode === 'create-destroy') for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (!hostId || !getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)) blockers.push(`Update the host agent for ${member.name} before enabling automatic removal.`);
for (const member of snapshot.members)
if (!['occupied', 'empty'].includes(member.state))
blockers.push(
`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`
);
if (policy.mode === 'create-destroy')
for (const member of snapshot.members) {
const hostId = getServer(member.name)?.host_id;
if (
!hostId ||
!getHost(hostId)?.inventory.capabilities?.includes(autoscalingRemovalCapability)
)
blockers.push(
`Update the host agent for ${member.name} before enabling automatic removal.`
);
}
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
)
blockers.push('Settle the current autoscaling operation before editing a live policy.');
try {
nativeLaunchSelection({ automatic: true }, policy.game);
} catch (error) {
blockers.push((error as Error).message);
}
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) blockers.push('Settle the current autoscaling operation before editing a live policy.');
try { nativeLaunchSelection({ automatic: true }, policy.game); } catch (error) { blockers.push((error as Error).message); }
}
const decision = planScaling(policy, snapshot, Date.now());
return { policy, snapshot, decision, blockers };
@@ -50,16 +122,48 @@ export function saveScaling(input: unknown, id?: string, revision?: number): Sca
return db().transaction(() => {
const evaluation = evaluateScaling(input, id, revision);
if (evaluation.blockers.length) throw new ScalingError(evaluation.blockers[0]);
const policy = evaluation.policy, settings = scalingSettings(policy), nextId = id ?? randomUUID(), at = new Date().toISOString();
const policy = evaluation.policy,
settings = scalingSettings(policy),
nextId = id ?? randomUUID(),
at = new Date().toISOString();
if (id) revokePolicyRemovals(id);
if (!id && db().prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?').get(policy.game, policy.mapSlug)) throw new ScalingError('A policy already exists for this scope.', 409);
db().prepare(`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`).run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
if (
!id &&
db()
.prepare('SELECT 1 FROM autoscaling_policies WHERE game=? AND map_slug=?')
.get(policy.game, policy.mapSlug)
)
throw new ScalingError('A policy already exists for this scope.', 409);
db()
.prepare(
`INSERT INTO autoscaling_policies(id,game,map_slug,settings,mode,updated_at) VALUES(?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET settings=excluded.settings,mode=excluded.mode,revision=autoscaling_policies.revision+1,paused_reason=NULL,updated_at=excluded.updated_at`
)
.run(nextId, policy.game, policy.mapSlug, JSON.stringify(settings), policy.mode, at);
// An edit starts a new observation interval; earlier inactivity is not authorization.
db().prepare('DELETE FROM autoscaling_members WHERE policy_id=?').run(nextId);
db().prepare('UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?').run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode)) for (const member of evaluation.snapshot.members) db().prepare('INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)')
.run(nextId, member.name, member.identity, JSON.stringify({ identity: member.identity, observedAt: member.observedAt, emptySince: null, evaluatedAt: Date.now() }));
db()
.prepare(
'UPDATE autoscaling_policies SET owns_map=CASE WHEN ? THEN 1 ELSE owns_map END WHERE id=?'
)
.run(liveScaling(policy.mode) ? 1 : 0, nextId);
if (liveScaling(policy.mode))
for (const member of evaluation.snapshot.members)
db()
.prepare(
'INSERT INTO autoscaling_members(policy_id,server,identity,observation,adopted) VALUES(?,?,?,?,1)'
)
.run(
nextId,
member.name,
member.identity,
JSON.stringify({
identity: member.identity,
observedAt: member.observedAt,
emptySince: null,
evaluatedAt: Date.now()
})
);
reconcileScaling(getScalingPolicy(nextId)!);
return getScalingPolicy(nextId)!;
})();
@@ -69,10 +173,18 @@ export function deleteScaling(id: string, revision: number): void {
db().transaction(() => {
const policy = getScalingPolicy(id);
if (!policy) throw new ScalingError('Policy not found.', 404);
if (policy.revision !== revision) throw new ScalingError('Policy changed. Refresh before deleting.', 409);
if (policy.revision !== revision)
throw new ScalingError('Policy changed. Refresh before deleting.', 409);
// Keep admission evidence and ownership while a host command may still act.
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(id)) {
throw new ScalingError('Pause this policy and settle its outstanding Provisioning work before deleting it.', 409);
if (
db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(id)
) {
throw new ScalingError(
'Pause this policy and settle its outstanding Provisioning work before deleting it.',
409
);
}
revokePolicyRemovals(id);
db().prepare('DELETE FROM autoscaling_authorizations WHERE policy_id=?').run(id);
@@ -85,39 +197,105 @@ export function deleteScaling(id: string, revision: number): void {
export function setScalingGlobalPause(paused: boolean): void {
db().transaction(() => {
db().prepare("INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(paused ? '1' : '0');
if (paused) for (const policy of listScalingPolicies()) { revokePolicyRemovals(policy.id); reconcileScaling(policy); }
db()
.prepare(
"INSERT INTO settings(key,value) VALUES('autoscaling_paused',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value"
)
.run(paused ? '1' : '0');
if (paused)
for (const policy of listScalingPolicies()) {
revokePolicyRemovals(policy.id);
reconcileScaling(policy);
}
})();
}
function reconcileScaling(policy: ScalingPolicy): void {
const intents = db().prepare(`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
const intents = db()
.prepare(
`SELECT a.id,a.revision,d.id AS operation_id,d.action,d.state,d.submitted_at,n.operation_id AS command_id
FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as
{ id: string; revision: number; operation_id: string; action: string; state: string; submitted_at: string | null; command_id: string | null }[];
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as {
id: string;
revision: number;
operation_id: string;
action: string;
state: string;
submitted_at: string | null;
command_id: string | null;
}[];
for (const intent of intents) {
if (intent.action === 'create' && ['queued','waiting','cancelled'].includes(intent.state) && !intent.command_id && !intent.submitted_at
&& (scalingAdmission(intent.operation_id) || intent.state === 'cancelled')) cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (
intent.action === 'create' &&
['queued', 'waiting', 'cancelled'].includes(intent.state) &&
!intent.command_id &&
!intent.submitted_at &&
(scalingAdmission(intent.operation_id) || intent.state === 'cancelled')
)
cancelCreation(intent.operation_id, 'autoscaling-withdrawal', false);
if (intent.action === 'destroy' && scalingAdmission(intent.operation_id)) {
revokePolicyRemovals(policy.id);
if (!intent.command_id) cancelRemoval(intent.operation_id, false);
}
const current = db().prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?').get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done','warning','failed','cancelled'].includes(current.state);
const command = intent.command_id ? db().prepare('SELECT state,result FROM ec_host_operations WHERE id=?').get(intent.command_id) as { state: string; result: string | null } | undefined : undefined;
const invalidated = command?.state === 'failed' && command.result && JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending = current.state === 'cancelled' && !current.server_deleted_at && command?.state === 'completed'
&& !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL').get(intent.operation_id);
const settled = (!intent.command_id || !!command && ['completed','failed'].includes(command.state)) && !receiptPending;
if (!invalidated && (current.state === 'failed' || current.state === 'interrupted' || command?.state === 'unknown') && liveScaling(policy.mode)) {
db().prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?').run(`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`, new Date().toISOString(), policy.id);
const current = db()
.prepare('SELECT state,server_deleted_at FROM server_deployments WHERE id=?')
.get(intent.operation_id) as { state: string; server_deleted_at: string | null };
const terminal = ['done', 'warning', 'failed', 'cancelled'].includes(current.state);
const command = intent.command_id
? (db()
.prepare('SELECT state,result FROM ec_host_operations WHERE id=?')
.get(intent.command_id) as { state: string; result: string | null } | undefined)
: undefined;
const invalidated =
command?.state === 'failed' &&
command.result &&
JSON.parse(command.result).autoscalingInvalidated === true;
const receiptPending =
current.state === 'cancelled' &&
!current.server_deleted_at &&
command?.state === 'completed' &&
!!db()
.prepare(
'SELECT 1 FROM native_provisioning WHERE deployment_id=? AND applied_at IS NULL'
)
.get(intent.operation_id);
const settled =
(!intent.command_id ||
(!!command && ['completed', 'failed'].includes(command.state))) &&
!receiptPending;
if (
!invalidated &&
(current.state === 'failed' ||
current.state === 'interrupted' ||
command?.state === 'unknown') &&
liveScaling(policy.mode)
) {
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=?,updated_at=? WHERE id=?')
.run(
`Inspect failed or uncertain Provisioning operation ${intent.operation_id} before resuming.`,
new Date().toISOString(),
policy.id
);
}
if (terminal && settled) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(intent.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled')) db().prepare("UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?").run(policy.id);
if (intent.action === 'destroy' && (invalidated || current.state === 'cancelled'))
db()
.prepare(
"UPDATE autoscaling_members SET observation=json_set(observation,'$.emptySince',NULL) WHERE policy_id=?"
)
.run(policy.id);
}
}
if (!liveScaling(policy.mode) && !db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) {
if (
!liveScaling(policy.mode) &&
!db()
.prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0')
.get(policy.id)
) {
db().prepare('UPDATE autoscaling_policies SET owns_map=0 WHERE id=?').run(policy.id);
}
}
@@ -131,48 +309,152 @@ export async function runAutoscaling(): Promise<boolean> {
if (policy.mode === 'off') continue;
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
if (scalingGloballyPaused())
snapshot.blockers.unshift('Autoscaling is globally paused.');
const pending = db()
.prepare(
`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.get(policy.id) as { id: string; stage: string } | undefined;
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
// Keep the observation interval that justified a pending removal;
// admission reads fresh evidence and revokes it on any new blocker.
if (member.state === 'removing' && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0').get(policy.id, member.operationId)) continue;
const observation = emptyObservation(member, { identity: member.identity, observedAt: member.observedAt, emptySince: member.emptySince, evaluatedAt: now }, now);
db().prepare(`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`).run(policy.id, member.name, member.identity, JSON.stringify(observation));
if (
member.state === 'removing' &&
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND operation_id=? AND resolved=0'
)
.get(policy.id, member.operationId)
)
continue;
const observation = emptyObservation(
member,
{
identity: member.identity,
observedAt: member.observedAt,
emptySince: member.emptySince,
evaluatedAt: now
},
now
);
db()
.prepare(
`INSERT INTO autoscaling_members(policy_id,server,identity,observation) VALUES(?,?,?,?)
ON CONFLICT(policy_id,server) DO UPDATE SET identity=excluded.identity,observation=excluded.observation`
)
.run(policy.id, member.name, member.identity, JSON.stringify(observation));
}
// Evidence includes provenance and timestamps but never keys or player rosters.
const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers });
const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex');
const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy');
const evidence = JSON.stringify({
members: snapshot.members,
reservations: snapshot.reservedNames,
keyPool: snapshot.keyPool,
blockers: snapshot.blockers
});
const fingerprint = createHash('sha256')
.update(
JSON.stringify([
policy.revision,
decision,
snapshot.members.map((m) => [m.name, m.identity, m.state])
])
)
.digest('hex');
const last = db()
.prepare(
'SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1'
)
.get(policy.id) as { id: string; fingerprint: string } | undefined;
const execute =
liveScaling(policy.mode) &&
(decision.action === 'create' || decision.action === 'destroy');
const decisionId = randomUUID();
if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id);
else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)')
.run(decisionId, policy.id, policy.revision, now, now, fingerprint, JSON.stringify(decision), evidence);
if (!execute && last?.fingerprint === fingerprint)
db()
.prepare(
'UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?'
)
.run(now, evidence, last.id);
else
db()
.prepare(
'INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)'
)
.run(
decisionId,
policy.id,
policy.revision,
now,
now,
fingerprint,
JSON.stringify(decision),
evidence
);
if (execute) {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?')
.run(decisionId);
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
const result =
decision.action === 'create'
? queueNativeCreationIntent({
server: decision.server!,
game: policy.game,
mapSlug: parseServerName(decision.server!)!.mapSlug,
selection: nativeLaunchSelection(
{ automatic: true },
policy.game
),
createdBy: `autoscaling:${policy.id}`,
autoscalingDecisionId: decisionId
})
: destroyServer(
decision.server!,
`autoscaling:${policy.id}`,
decisionId
);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
if (!result.deploymentId)
throw new Error('Provisioning did not return an operation identity.');
db()
.prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?')
.run(result.deploymentId, decisionId);
db()
.prepare(
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)"
)
.run(result.deploymentId, policy.id, decision.server);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
db().prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?').run(`Scaling action could not be queued: ${(error as Error).message}`, policy.id);
db()
.prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?')
.run(decisionId);
db()
.prepare('UPDATE autoscaling_policies SET paused_reason=? WHERE id=?')
.run(
`Scaling action could not be queued: ${(error as Error).message}`,
policy.id
);
}
}
db().prepare(`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`)
.run(policy.id, Math.floor(now / 15_000) * 15_000, policy.mode, decision.occupied, decision.empty, decision.unknown, decision.deficit);
db()
.prepare(
`INSERT OR IGNORE INTO autoscaling_samples(policy_id,sampled_at,mode,occupied,empty,unknown,deficit) VALUES(?,?,?,?,?,?,?)`
)
.run(
policy.id,
Math.floor(now / 15_000) * 15_000,
policy.mode,
decision.occupied,
decision.empty,
decision.unknown,
decision.deficit
);
})();
}
return submitted;
@@ -180,37 +462,96 @@ export async function runAutoscaling(): Promise<boolean> {
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
db()
.prepare(
`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`
)
.run(cutoff);
db()
.prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1')
.run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
export function scalingReport(): ScalingReport {
const policies: ScalingView[] = listScalingPolicies().map(policy => {
const policies: ScalingView[] = listScalingPolicies().map((policy) => {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
const history = (
db()
.prepare(
`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`
)
.all(policy.id) as {
id: string;
evaluated_at: number;
last_seen_at: number;
decision: string;
operation_id: string | null;
operation_state: string | null;
}[]
).map((row) => ({
id: row.id,
at: row.evaluated_at,
lastSeenAt: row.last_seen_at,
decision: JSON.parse(row.decision),
operationId: row.operation_id,
operationState: row.operation_state
}));
const unsettled = db()
.prepare(
`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`
)
.all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(
db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as {
owns_map: number;
}
).owns_map;
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
const totals = db()
.prepare(
`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`
)
.get(policy.id) as ScalingView['totals'];
const operations = db()
.prepare(
`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
return { policy, snapshot, decision, blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`
)
.get(
policy.id,
new Date(Date.now() - 31 * 86400_000).toISOString()
) as ScalingView['operations'];
return {
policy,
snapshot,
decision,
blockers: [],
history,
lastEvaluatedAt: history[0]?.lastSeenAt ?? null,
totals,
operations,
unsettled,
ownsMap
};
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
return {
policies,
globalPaused: scalingGloballyPaused(),
liveAvailable: true,
removalAvailable: true
};
}
src/lib/server/autoscaling/ownership.ts
+36 -8
View File
@@ -4,18 +4,29 @@ import { liveScaling, type ScalingPolicy } from '../../autoscaling.js';
import { mapIdentity } from './snapshot.js';
function owningPolicy(game: string, mapSlug: string) {
return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1")
.get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
return db()
.prepare(
"SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1"
)
.get(game, mapSlug, mapSlug) as
{ id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
}
/** Durable ownership prevents overlapping policies even while scaling is paused. */
export function scalingOwner(game: string, mapSlug: string): string | null {
return owningPolicy(game, mapSlug)?.id ?? null;
}
const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
const ownershipMessage = (owner: string) =>
`Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
function manualLineupBlocker(game: string, mapSlug: string): string | null {
const owner = owningPolicy(game, mapSlug);
if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null;
if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null;
if (
(
db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as
{ value: string } | undefined
)?.value === '1'
)
return null;
return ownershipMessage(owner.id);
}
export function scalingOwnershipBlocker(server: string): string | null {
@@ -30,13 +41,30 @@ export function assertScalingOwnership(game: string, mapSlug: string, decisionId
}
const owner = scalingOwner(game, mapSlug);
if (!owner) throw new Error('Autoscaling no longer owns this map.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
if (
db()
.prepare(
'SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL'
)
.get(decisionId, owner)
)
return;
throw new Error(ownershipMessage(owner));
}
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
export function pauseScalingForOperation(operationId: string): void {
const row = db().prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?').get(operationId) as { policy_id: string } | undefined;
const row = db()
.prepare('SELECT policy_id FROM autoscaling_decisions WHERE operation_id=?')
.get(operationId) as { policy_id: string } | undefined;
if (!row) return;
db().prepare("UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?").run(new Date().toISOString(), row.policy_id);
db().prepare('UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL').run(Date.now(), row.policy_id);
db()
.prepare(
"UPDATE autoscaling_policies SET mode='off',revision=revision+1,paused_reason='Paused after operator cancellation.',updated_at=? WHERE id=?"
)
.run(new Date().toISOString(), row.policy_id);
db()
.prepare(
'UPDATE autoscaling_authorizations SET revoked_at=? WHERE policy_id=? AND claimed_at IS NULL'
)
.run(Date.now(), row.policy_id);
}
src/lib/server/autoscaling/planner.ts
+143 -45
View File
@@ -1,72 +1,170 @@
import type { ScalingDecision, ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import type {
ScalingDecision,
ScalingMember,
ScalingPolicy,
ScalingSnapshot
} from '../../autoscaling.js';
import { parseServerName, zombieMaps } from '../../map-aliases.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
export interface EmptyObservation { identity: string; observedAt: number | null; emptySince: number | null; evaluatedAt: number }
export interface EmptyObservation {
identity: string;
observedAt: number | null;
emptySince: number | null;
evaluatedAt: number;
}
/** Only distinct, continuous evidence can advance an empty interval. */
export function emptyObservation(member: ScalingMember, previous: EmptyObservation | undefined, now: number): EmptyObservation {
export function emptyObservation(
member: ScalingMember,
previous: EmptyObservation | undefined,
now: number
): EmptyObservation {
const stamp = member.observedAt;
const empty = member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous = empty && previous?.identity === member.identity && previous.emptySince !== null
&& (!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt)
&& previous.observedAt !== null && stamp >= previous.observedAt && stamp - previous.observedAt <= ONLINE_MAX_MS
&& now >= previous.evaluatedAt && now - previous.evaluatedAt <= ONLINE_MAX_MS;
return { identity: member.identity, observedAt: stamp, emptySince: empty ? continuous ? previous!.emptySince : stamp : null, evaluatedAt: now };
const empty =
member.state === 'empty' && stamp !== null && stamp <= now && now - stamp <= ONLINE_MAX_MS;
const continuous =
empty &&
previous?.identity === member.identity &&
previous.emptySince !== null &&
(!member.emptyStartedAt || previous.emptySince >= member.emptyStartedAt) &&
previous.observedAt !== null &&
stamp >= previous.observedAt &&
stamp - previous.observedAt <= ONLINE_MAX_MS &&
now >= previous.evaluatedAt &&
now - previous.evaluatedAt <= ONLINE_MAX_MS;
return {
identity: member.identity,
observedAt: stamp,
emptySince: empty ? (continuous ? previous!.emptySince : stamp) : null,
evaluatedAt: now
};
}
export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, now: number): ScalingDecision {
export function planScaling(
policy: ScalingPolicy,
snapshot: ScalingSnapshot,
now: number
): ScalingDecision {
if (policy.mapSlug === '*') {
const maps = zombieMaps(policy.game);
const decisions = maps.map(map => {
const matches = (name: string) => (snapshot.members.find(m => m.name === name)?.mapSlug ?? parseServerName(name)?.mapSlug) === map.slug;
return planScaling({ ...policy, mapSlug: map.slug }, {
...snapshot, members: snapshot.members.filter(m => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
}, now);
const decisions = maps.map((map) => {
const matches = (name: string) =>
(snapshot.members.find((m) => m.name === name)?.mapSlug ??
parseServerName(name)?.mapSlug) === map.slug;
return planScaling(
{ ...policy, mapSlug: map.slug },
{
...snapshot,
members: snapshot.members.filter((m) => matches(m.name)),
reservedNames: snapshot.reservedNames.filter(matches)
},
now
);
});
// Serialize game-wide work and prefer replenishing any map before trimming another.
const pending = snapshot.members.find(m => m.state === 'creating' || m.state === 'removing');
const selected = decisions.find(d => d.action === 'create') ?? decisions.find(d => d.action === 'destroy')
?? decisions.find(d => d.deficit > 0 || d.unknown > 0) ?? decisions[0];
const pending = snapshot.members.find(
(m) => m.state === 'creating' || m.state === 'removing'
);
const selected =
decisions.find((d) => d.action === 'create') ??
decisions.find((d) => d.action === 'destroy') ??
decisions.find((d) => d.deficit > 0 || d.unknown > 0) ??
decisions[0];
const map = maps[decisions.indexOf(selected)];
return { ...selected, action: pending ? 'wait' : selected.action, server: pending ? null : selected.server,
reason: pending ? pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.` : `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0), deficit: decisions.reduce((n, d) => n + d.deficit, 0), occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0), pending: decisions.reduce((n, d) => n + d.pending, 0), unknown: decisions.reduce((n, d) => n + d.unknown, 0) };
return {
...selected,
action: pending ? 'wait' : selected.action,
server: pending ? null : selected.server,
reason: pending
? (pending.blocker ?? `Waiting for ${pending.name}: ${pending.state}.`)
: `${map.friendly}: ${selected.reason}`,
desired: decisions.reduce((n, d) => n + d.desired, 0),
deficit: decisions.reduce((n, d) => n + d.deficit, 0),
occupied: decisions.reduce((n, d) => n + d.occupied, 0),
empty: decisions.reduce((n, d) => n + d.empty, 0),
pending: decisions.reduce((n, d) => n + d.pending, 0),
unknown: decisions.reduce((n, d) => n + d.unknown, 0)
};
}
const { members } = snapshot;
const occupied = members.filter(m => m.state === 'occupied').length, empty = members.filter(m => m.state === 'empty').length;
const pending = members.filter(m => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter(m => m.state === 'unknown' || m.state === 'stopped').length;
const occupied = members.filter((m) => m.state === 'occupied').length,
empty = members.filter((m) => m.state === 'empty').length;
const pending = members.filter((m) => m.state === 'creating' || m.state === 'removing').length;
const unknown = members.filter((m) => m.state === 'unknown' || m.state === 'stopped').length;
const desired = Math.min(policy.maximum, Math.max(policy.minimum, occupied + policy.spares));
const result = (action: ScalingDecision['action'], reason: string, server: string | null = null): ScalingDecision =>
({ action, reason, server, desired, deficit: Math.max(0, policy.spares - empty), occupied, empty, pending, unknown });
const result = (
action: ScalingDecision['action'],
reason: string,
server: string | null = null
): ScalingDecision => ({
action,
reason,
server,
desired,
deficit: Math.max(0, policy.spares - empty),
occupied,
empty,
pending,
unknown
});
if (policy.mode === 'off') return result('wait', policy.pausedReason ?? 'Policy is off.');
if (policy.pausedReason) return result('wait', policy.pausedReason);
if (snapshot.blockers.length) return result('wait', snapshot.blockers[0]);
const blocked = members.find(m => !['occupied', 'empty'].includes(m.state));
if (blocked) return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (members.some(m => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS)) return result('wait', 'Waiting for fresh occupancy evidence.');
const blocked = members.find((m) => !['occupied', 'empty'].includes(m.state));
if (blocked)
return result('wait', blocked.blocker ?? `Waiting for ${blocked.name}: ${blocked.state}.`);
if (
members.some(
(m) => m.observedAt === null || m.observedAt > now || now - m.observedAt > ONLINE_MAX_MS
)
)
return result('wait', 'Waiting for fresh occupancy evidence.');
if (occupied + empty < desired) {
if (snapshot.reservedNames.length >= policy.maximum) return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available) return result('wait', snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`);
const available = Array.from({ length: 20 }, (_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`)
.find(name => !snapshot.reservedNames.includes(name));
return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.');
if (snapshot.reservedNames.length >= policy.maximum)
return result('wait', 'Maximum lobby reservations reached.');
if (!snapshot.keyPool.available)
return result(
'wait',
snapshot.keyPool.total
? `All ${policy.game.toUpperCase()} Plutonium keys are in use. Waiting for a free key.`
: `No Plutonium keys are saved for ${policy.game.toUpperCase()}.`
);
const available = Array.from(
{ length: 20 },
(_, index) => `${policy.game}_zombie_${policy.mapSlug}_${index + 1}`
).find((name) => !snapshot.reservedNames.includes(name));
return available
? result('create', 'Replenish empty lobby capacity.', available)
: result('wait', 'No unused lobby identity is available.');
}
if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) {
// Protect the lowest-numbered spares before checking removal timers.
// Lobby 1 must survive even when a higher lobby has only just emptied.
const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER)
|| a.name.localeCompare(b.name, 'en', { numeric: true }));
const orderedEmpty = members
.filter((m) => m.state === 'empty')
.sort(
(a, b) =>
(parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) -
(parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) ||
a.name.localeCompare(b.name, 'en', { numeric: true })
);
const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied);
const candidates = orderedEmpty.slice(requiredEmpty).reverse()
.filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000);
if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
const candidates = orderedEmpty
.slice(requiredEmpty)
.reverse()
.filter(
(m) =>
m.emptySince !== null &&
m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000
);
if (candidates.length)
return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name);
return result('wait', 'Waiting for a continuous empty interval.');
}
return result('wait', empty < policy.spares ? 'Maximum reached; the empty spare target cannot be met.' : 'Empty spare target is satisfied.');
return result(
'wait',
empty < policy.spares
? 'Maximum reached; the empty spare target cannot be met.'
: 'Empty spare target is satisfied.'
);
}
src/lib/server/autoscaling/schema.ts
+13 -5
View File
@@ -36,20 +36,28 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
`);
handle.transaction(() => {
const columns = handle.pragma('table_info(autoscaling_policies)') as { name: string }[];
if (columns.some(column => column.name === 'next_action_at')) {
if (columns.some((column) => column.name === 'next_action_at')) {
handle.exec('ALTER TABLE autoscaling_policies DROP COLUMN next_action_at');
}
handle.exec(`UPDATE autoscaling_policies SET settings=json_remove(settings,'$.cooldownMinutes')
WHERE json_type(settings,'$.cooldownMinutes') IS NOT NULL`);
// Removed modes must not silently gain permission to destroy lobbies.
// Keep ownership while delivered work settles through normal reconciliation.
handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=?
handle
.prepare(
`UPDATE autoscaling_authorizations SET revoked_at=?
WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now());
handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
(SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`
)
.run(Date.now());
handle
.prepare(
`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'),
revision=revision+1, paused_reason='This autoscaling mode was removed. Review and turn autoscaling On to resume.',
owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0)
THEN owns_map ELSE 0 END, updated_at=?
WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString());
WHERE mode IN ('dry-run','create-only')`
)
.run(new Date().toISOString());
})();
}
src/lib/server/autoscaling/snapshot.ts
+189 -45
View File
@@ -7,74 +7,218 @@ import { freshStatusTimestamp } from '../../server-freshness.js';
import type { ScalingMember, ScalingPolicy, ScalingSnapshot } from '../../autoscaling.js';
import { emptyObservation, type EmptyObservation } from './planner.js';
export function mapIdentity(server: { name: string; game?: string | null; map_slug?: string | null; map?: string | null }): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name), game = server.game ?? parsed?.game;
export function mapIdentity(server: {
name: string;
game?: string | null;
map_slug?: string | null;
map?: string | null;
}): { game: string; mapSlug: string } | null {
const parsed = parseServerName(server.name),
game = server.game ?? parsed?.game;
if (!game) return null;
if (server.map_slug && zombieMaps(game).some(m => m.slug === server.map_slug)) return { game, mapSlug: server.map_slug };
if (server.map_slug && zombieMaps(game).some((m) => m.slug === server.map_slug))
return { game, mapSlug: server.map_slug };
if (server.map_slug) return null;
if (parsed?.game === game && parsed.mapCanonical && (!server.map || parsed.mapCanonical === server.map)) return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter(m => m.canonical === server.map);
if (
parsed?.game === game &&
parsed.mapCanonical &&
(!server.map || parsed.mapCanonical === server.map)
)
return { game, mapSlug: parsed.mapSlug };
const matches = zombieMaps(game).filter((m) => m.canonical === server.map);
return matches.length === 1 ? { game, mapSlug: matches[0].slug } : null;
}
export function protectedWork(server: string, exceptOperation = ''): string | null {
const handle = db();
if (handle.prepare(`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).get(server)) return 'Active beta';
if (handle.prepare(`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`).get(server)) return 'Campaign work';
if (handle.prepare(`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`).get(server)) return 'Maintenance work';
if (handle.prepare(`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`).get(server, exceptOperation)) return 'Unresolved recovery';
if (
handle
.prepare(
`SELECT 1 FROM beta_servers s JOIN betas b ON b.id=s.beta_id WHERE s.server=? AND s.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
)
.get(server)
)
return 'Active beta';
if (
handle
.prepare(
`SELECT 1 FROM deployment_servers WHERE server=? AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held','manual')`
)
.get(server)
)
return 'Campaign work';
if (
handle
.prepare(
`SELECT 1 FROM maintenance_jobs WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','restarting')`
)
.get(server)
)
return 'Maintenance work';
if (
handle
.prepare(
`SELECT 1 FROM restart_rollout WHERE server=? AND healthy_at IS NULL AND (operation_id IS NULL OR operation_id!=?)`
)
.get(server, exceptOperation)
)
return 'Unresolved recovery';
return null;
}
export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptOperation = ''): ScalingSnapshot {
const handle = db(), observe = observationReader();
export function scalingSnapshot(
policy: ScalingPolicy,
now = Date.now(),
exceptOperation = ''
): ScalingSnapshot {
const handle = db(),
observe = observationReader();
const rows = listServers();
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && (policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug); };
const matches = (value: Parameters<typeof mapIdentity>[0]) => {
const identity = mapIdentity(value);
return (
identity?.game === policy.game &&
(policy.mapSlug === '*' || identity.mapSlug === policy.mapSlug)
);
};
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
for (const row of rows)
if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game)
blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle
.prepare(
`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`
)
.all(exceptOperation) as {
id: string;
server: string;
action: string;
state: string;
stage: string;
}[];
const reservations = handle
.prepare(
`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`
)
.all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
const observation = observe(row), operation = operations.find(o => o.server === row.name);
const observation = observe(row),
operation = operations.find((o) => o.server === row.name);
const protection = protectedWork(row.name, exceptOperation);
const host = row.host_id ? handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as { mode: string } | undefined : undefined;
const host = row.host_id
? (handle.prepare('SELECT mode FROM ec_hosts WHERE id=?').get(row.host_id) as
{ mode: string } | undefined)
: undefined;
const identity = `${row.native_instance_id ?? row.created_at}:${row.instance_generation ?? 0}:${observation.telemetry_after ?? ''}`;
let state: ScalingMember['state'] = 'unknown', blocker: string | null = null;
let state: ScalingMember['state'] = 'unknown',
blocker: string | null = null;
let players = observation.health === 'online' ? observation.last_players : null;
let observedAt = players !== null && observation.last_seen_at ? Date.parse(observation.last_seen_at) : null;
let observedAt =
players !== null && observation.last_seen_at
? Date.parse(observation.last_seen_at)
: null;
let source = observation.status_source;
// Display evidence prefers telemetry; scaling additionally honors a current positive RCON veto.
if ((row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
&& (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) {
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
if (
(row.last_players ?? 0) > 0 &&
freshStatusTimestamp(row.last_seen_at, now) &&
(!observation.telemetry_after ||
Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))
) {
players = row.last_players;
observedAt = Date.parse(row.last_seen_at!);
source = 'rcon';
}
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, mapSlug: mapIdentity(row)!.mapSlug, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
if (protection || !row.native_instance_id || host?.mode !== 'active') {
state = 'protected';
blocker = protection ?? 'Native instance on an active host required';
} else if (operation) {
state =
operation.action === 'create'
? 'creating'
: operation.action === 'destroy'
? 'removing'
: 'protected';
blocker = `Provisioning ${operation.id}: ${operation.stage}`;
} else if (
row.desired_state === 'stopped' ||
['stopped', 'exited', 'created', 'dead'].includes(row.container_state ?? '')
)
state = 'stopped';
else if (
observation.health === 'online' &&
players !== null &&
observedAt !== null &&
observedAt <= now &&
freshStatusTimestamp(new Date(observedAt).toISOString(), now)
)
state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since]
.map((at) => Date.parse(at ?? ''))
.filter((at) => Number.isFinite(at) && at <= now);
return {
name: row.name,
mapSlug: mapIdentity(row)!.mapSlug,
identity,
state,
players,
observedAt,
source: source ?? null,
emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null,
blocker,
operationId: operation?.id ?? null
};
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, mapSlug: row.map_slug, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const row of reservations.filter(matches))
if (!members.some((m) => m.name === row.name)) {
const operation = operations.find((o) => o.server === row.name);
members.push({
name: row.name,
mapSlug: row.map_slug,
identity: 'reserved',
state: 'creating',
players: null,
observedAt: null,
source: null,
emptySince: null,
blocker: operation?.stage ?? 'Creation is pending.',
operationId: operation?.id ?? null
});
}
const retained = handle
.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?')
.all(policy.id) as { server: string; observation: string }[];
for (const member of members) {
const saved = retained.find(r => r.server === member.name);
const previous = saved ? JSON.parse(saved.observation) as EmptyObservation : undefined;
const saved = retained.find((r) => r.server === member.name);
const previous = saved ? (JSON.parse(saved.observation) as EmptyObservation) : undefined;
member.emptySince = emptyObservation(member, previous, now).emptySince;
}
const reservedNames = new Set([...rows.filter(matches).map(r => r.name), ...reservations.filter(matches).map(r => r.name)]);
for (const row of handle.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL').all() as { name: string }[]) if (matches(row)) {
reservedNames.add(row.name);
if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as { name: string }[]) if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperation) as { server: string } | undefined)?.server : undefined;
const reservedNames = new Set([
...rows.filter(matches).map((r) => r.name),
...reservations.filter(matches).map((r) => r.name)
]);
for (const row of handle
.prepare('SELECT server AS name FROM native_instances WHERE retired_at IS NULL')
.all() as { name: string }[])
if (matches(row)) {
reservedNames.add(row.name);
if (!members.some((m) => m.name === row.name))
blockers.push(`Unresolved native instance ${row.name}.`);
}
for (const row of handle.prepare('SELECT server_id AS name FROM server_key_leases').all() as {
name: string;
}[])
if (matches(row)) reservedNames.add(row.name);
const ownServer = exceptOperation
? (
handle
.prepare('SELECT server FROM server_deployments WHERE id=?')
.get(exceptOperation) as { server: string } | undefined
)?.server
: undefined;
const { total, available } = serverKeyPool(policy.game, ownServer);
return { members, reservedNames: [...reservedNames], keyPool: { total, available }, blockers };
}