josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Finish autoscaling diagnostics and document pilot rollout

Browse Source
This commit is contained in:
josh
2026-10-06 23:18:39 -04:00
parent e271f31325
commit a61eccaed6
10 changed files with 135 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/autoscaling/index.ts
+29 -16
View File
@@ -114,7 +114,8 @@ export async function runAutoscaling(): Promise<boolean> {
db().transaction(() => {
const snapshot = scalingSnapshot(policy, now);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.');
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
const decision = planScaling(policy, snapshot, now);
for (const member of snapshot.members) {
// Keep the observation interval that justified a pending removal;
@@ -137,16 +138,16 @@ export async function runAutoscaling(): Promise<boolean> {
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
try {
db().transaction(() => {
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
submitted = true;
const result = decision.action === 'create'
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
if (!result.ok) throw new Error(result.message);
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
submitted = true;
})();
} catch (error) {
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
@@ -162,7 +163,8 @@ export async function runAutoscaling(): Promise<boolean> {
export function pruneAutoscaling(now = Date.now()): void {
const cutoff = now - 31 * 86400_000;
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1 AND operation_id IS NULL').run(cutoff);
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
}
@@ -171,16 +173,27 @@ export function scalingReport(): ScalingReport {
const snapshot = scalingSnapshot(policy);
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
const decision = planScaling(policy, snapshot, Date.now());
const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id }));
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
});
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
}
src/lib/server/autoscaling/schema.ts
+2
View File
@@ -22,6 +22,7 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
);
CREATE UNIQUE INDEX IF NOT EXISTS autoscaling_one_intent ON autoscaling_decisions(policy_id) WHERE resolved=0;
CREATE INDEX IF NOT EXISTS autoscaling_history ON autoscaling_decisions(policy_id,evaluated_at DESC);
CREATE INDEX IF NOT EXISTS autoscaling_decision_retention ON autoscaling_decisions(last_seen_at) WHERE resolved=1;
CREATE TABLE IF NOT EXISTS autoscaling_samples (
policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), sampled_at INTEGER NOT NULL,
mode TEXT NOT NULL, occupied INTEGER NOT NULL, empty INTEGER NOT NULL, unknown INTEGER NOT NULL,
@@ -31,5 +32,6 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
operation_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL,
expires_at INTEGER NOT NULL, claimed_at INTEGER, revoked_at INTEGER
);
CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at);
`);
}
src/lib/server/autoscaling/snapshot.ts
+7 -4
View File
@@ -31,8 +31,8 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
const blockers: string[] = [];
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
const operations = handle.prepare(`SELECT id,server,action,state FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string }[];
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
@@ -50,14 +50,17 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
}
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id} is ${operation.state}`; }
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
});
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: 'Creation is pending.', operationId: operations.find(o => o.server === row.name)?.id ?? null });
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
const operation = operations.find(o => o.server === row.name);
members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
}
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
for (const member of members) {
const saved = retained.find(r => r.server === member.name);
src/lib/server/provisioning/native.ts
+8
View File
@@ -500,6 +500,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
if (operation.state === 'failed') {
if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) {
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?")
.run(new Date().toISOString(), job.id);
recordProgress(job.id, { blocker: null, event: { kind: 'outcome', message: 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', details: { state: 'cancelled' } } });
})();
return;
}
if (migration) {
if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
db().transaction(() => {