Finish autoscaling diagnostics and document pilot rollout
This commit is contained in:
@@ -23,10 +23,13 @@ export interface ScalingDecision {
|
||||
desired: number; occupied: number; empty: number; pending: number; unknown: number;
|
||||
}
|
||||
export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] }
|
||||
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null }
|
||||
export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
|
||||
export interface ScalingView extends ScalingPreview {
|
||||
history: ScalingHistory[]; lastEvaluatedAt: number | null;
|
||||
totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
|
||||
operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null };
|
||||
ownsMap: boolean;
|
||||
unsettled: { id: string; action: string; state: string; stage: string }[];
|
||||
}
|
||||
export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
|
||||
export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy';
|
||||
|
||||
@@ -114,7 +114,8 @@ export async function runAutoscaling(): Promise<boolean> {
|
||||
db().transaction(() => {
|
||||
const snapshot = scalingSnapshot(policy, now);
|
||||
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
|
||||
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.');
|
||||
const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
|
||||
if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
|
||||
const decision = planScaling(policy, snapshot, now);
|
||||
for (const member of snapshot.members) {
|
||||
// Keep the observation interval that justified a pending removal;
|
||||
@@ -137,16 +138,16 @@ export async function runAutoscaling(): Promise<boolean> {
|
||||
db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
|
||||
try {
|
||||
db().transaction(() => {
|
||||
const result = decision.action === 'create'
|
||||
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
|
||||
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
|
||||
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
|
||||
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
|
||||
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
|
||||
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
|
||||
submitted = true;
|
||||
const result = decision.action === 'create'
|
||||
? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
|
||||
selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
|
||||
: destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
|
||||
db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
|
||||
db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
|
||||
db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
|
||||
submitted = true;
|
||||
})();
|
||||
} catch (error) {
|
||||
db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
|
||||
@@ -162,7 +163,8 @@ export async function runAutoscaling(): Promise<boolean> {
|
||||
|
||||
export function pruneAutoscaling(now = Date.now()): void {
|
||||
const cutoff = now - 31 * 86400_000;
|
||||
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1 AND operation_id IS NULL').run(cutoff);
|
||||
db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
|
||||
db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
|
||||
db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
|
||||
}
|
||||
|
||||
@@ -171,16 +173,27 @@ export function scalingReport(): ScalingReport {
|
||||
const snapshot = scalingSnapshot(policy);
|
||||
if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
|
||||
const decision = planScaling(policy, snapshot, Date.now());
|
||||
const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as
|
||||
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[])
|
||||
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id }));
|
||||
const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
|
||||
FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
|
||||
WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
|
||||
{ id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
|
||||
.map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
|
||||
const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
|
||||
const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
|
||||
// Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
|
||||
const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
|
||||
FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
|
||||
COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
|
||||
COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
|
||||
COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
|
||||
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
|
||||
const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
|
||||
COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
|
||||
COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
|
||||
AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
|
||||
MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
|
||||
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
|
||||
WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
|
||||
return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
|
||||
});
|
||||
return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS autoscaling_one_intent ON autoscaling_decisions(policy_id) WHERE resolved=0;
|
||||
CREATE INDEX IF NOT EXISTS autoscaling_history ON autoscaling_decisions(policy_id,evaluated_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS autoscaling_decision_retention ON autoscaling_decisions(last_seen_at) WHERE resolved=1;
|
||||
CREATE TABLE IF NOT EXISTS autoscaling_samples (
|
||||
policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), sampled_at INTEGER NOT NULL,
|
||||
mode TEXT NOT NULL, occupied INTEGER NOT NULL, empty INTEGER NOT NULL, unknown INTEGER NOT NULL,
|
||||
@@ -31,5 +32,6 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
|
||||
operation_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL,
|
||||
expires_at INTEGER NOT NULL, claimed_at INTEGER, revoked_at INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at);
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -31,8 +31,8 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
|
||||
const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
|
||||
const blockers: string[] = [];
|
||||
for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
|
||||
const operations = handle.prepare(`SELECT id,server,action,state FROM server_deployments WHERE server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string }[];
|
||||
const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
|
||||
const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
|
||||
JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
|
||||
const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
|
||||
@@ -50,14 +50,17 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
|
||||
players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
|
||||
}
|
||||
if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
|
||||
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id} is ${operation.state}`; }
|
||||
else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
|
||||
else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
|
||||
else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
|
||||
const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
|
||||
return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
|
||||
emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
|
||||
});
|
||||
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: 'Creation is pending.', operationId: operations.find(o => o.server === row.name)?.id ?? null });
|
||||
for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
|
||||
const operation = operations.find(o => o.server === row.name);
|
||||
members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
|
||||
}
|
||||
const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
|
||||
for (const member of members) {
|
||||
const saved = retained.find(r => r.server === member.name);
|
||||
|
||||
@@ -500,6 +500,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
}
|
||||
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
|
||||
if (operation.state === 'failed') {
|
||||
if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) {
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?")
|
||||
.run(new Date().toISOString(), job.id);
|
||||
recordProgress(job.id, { blocker: null, event: { kind: 'outcome', message: 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', details: { state: 'cancelled' } } });
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (migration) {
|
||||
if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
|
||||
db().transaction(() => {
|
||||
|
||||
Reference in New Issue
Block a user