Finish autoscaling diagnostics and document pilot rollout

src/lib/autoscaling.ts

@@ -23,10 +23,13 @@ export interface ScalingDecision {
     desired: number; occupied: number; empty: number; pending: number; unknown: number;
 }
 export interface ScalingPreview { policy: ScalingPolicy; snapshot: ScalingSnapshot; decision: ScalingDecision; token: string; blockers: string[] }
-export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null }
+export interface ScalingHistory { id: string; at: number; lastSeenAt: number; decision: ScalingDecision; operationId: string | null; operationState: string | null }
 export interface ScalingView extends ScalingPreview {
     history: ScalingHistory[]; lastEvaluatedAt: number | null;
     totals: { observedMinutes: number; emptyLobbyHours: number; deficitMinutes: number; unknownMinutes: number };
+    operations: { creates: number; removals: number; cancelled: number; failed: number; readinessSamples: number; readinessMeanSeconds: number | null; readinessMaxSeconds: number | null };
+    ownsMap: boolean;
+    unsettled: { id: string; action: string; state: string; stage: string }[];
 }
 export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean }
 export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy';

src/lib/server/autoscaling/index.ts

@@ -114,7 +114,8 @@ export async function runAutoscaling(): Promise<boolean> {
         db().transaction(() => {
             const snapshot = scalingSnapshot(policy, now);
             if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
-            if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE policy_id=? AND resolved=0').get(policy.id)) snapshot.blockers.unshift('Waiting for the current autoscaling operation to settle.');
+            const pending = db().prepare(`SELECT d.id,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).get(policy.id) as { id: string; stage: string } | undefined;
+            if (pending) snapshot.blockers.unshift(`Provisioning ${pending.id}: ${pending.stage}`);
             const decision = planScaling(policy, snapshot, now);
             for (const member of snapshot.members) {
                 // Keep the observation interval that justified a pending removal;
@@ -137,16 +138,16 @@ export async function runAutoscaling(): Promise<boolean> {
                 db().prepare('UPDATE autoscaling_decisions SET resolved=0 WHERE id=?').run(decisionId);
                 try {
                     db().transaction(() => {
-                    const result = decision.action === 'create'
-                        ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
-                            selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
-                        : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
-                    if (!result.ok) throw new Error(result.message);
-                    if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
-                    db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
-                    db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
-                    db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
-                    submitted = true;
+                        const result = decision.action === 'create'
+                            ? queueNativeCreationIntent({ server: decision.server!, game: policy.game, mapSlug: policy.mapSlug,
+                                selection: nativeLaunchSelection({ automatic: true }, policy.game), createdBy: `autoscaling:${policy.id}`, autoscalingDecisionId: decisionId })
+                            : destroyServer(decision.server!, `autoscaling:${policy.id}`, decisionId);
+                        if (!result.ok) throw new Error(result.message);
+                        if (!result.deploymentId) throw new Error('Provisioning did not return an operation identity.');
+                        db().prepare('UPDATE autoscaling_decisions SET operation_id=? WHERE id=?').run(result.deploymentId, decisionId);
+                        db().prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES(?,'autoscaling',?,?)").run(result.deploymentId, policy.id, decision.server);
+                        db().prepare('UPDATE autoscaling_policies SET next_action_at=? WHERE id=?').run(now + policy.cooldownMinutes * 60_000, policy.id);
+                        submitted = true;
                     })();
                 } catch (error) {
                     db().prepare('UPDATE autoscaling_decisions SET resolved=1 WHERE id=?').run(decisionId);
@@ -162,7 +163,8 @@ export async function runAutoscaling(): Promise<boolean> {
 
 export function pruneAutoscaling(now = Date.now()): void {
     const cutoff = now - 31 * 86400_000;
-    db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1 AND operation_id IS NULL').run(cutoff);
+    db().prepare(`DELETE FROM autoscaling_authorizations WHERE operation_id IN (SELECT operation_id FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1)`).run(cutoff);
+    db().prepare('DELETE FROM autoscaling_decisions WHERE last_seen_at<? AND resolved=1').run(cutoff);
     db().prepare('DELETE FROM autoscaling_samples WHERE sampled_at<?').run(cutoff);
 }
 
@@ -171,16 +173,27 @@ export function scalingReport(): ScalingReport {
         const snapshot = scalingSnapshot(policy);
         if (scalingGloballyPaused()) snapshot.blockers.unshift('Autoscaling is globally paused.');
         const decision = planScaling(policy, snapshot, Date.now());
-        const history = (db().prepare('SELECT id,evaluated_at,last_seen_at,decision,operation_id FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 30').all(policy.id) as
-            { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null }[])
-            .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id }));
+        const history = (db().prepare(`SELECT a.id,a.evaluated_at,a.last_seen_at,a.decision,a.operation_id,d.state AS operation_state
+            FROM autoscaling_decisions a LEFT JOIN server_deployments d ON d.id=a.operation_id
+            WHERE a.policy_id=? ORDER BY a.evaluated_at DESC,a.rowid DESC LIMIT 30`).all(policy.id) as
+            { id: string; evaluated_at: number; last_seen_at: number; decision: string; operation_id: string | null; operation_state: string | null }[])
+            .map(row => ({ id: row.id, at: row.evaluated_at, lastSeenAt: row.last_seen_at, decision: JSON.parse(row.decision), operationId: row.operation_id, operationState: row.operation_state }));
+        const unsettled = db().prepare(`SELECT d.id,d.action,d.state,d.stage FROM autoscaling_decisions a JOIN server_deployments d ON d.id=a.operation_id WHERE a.policy_id=? AND a.resolved=0`).all(policy.id) as ScalingView['unsettled'];
+        const ownsMap = !!(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(policy.id) as { owns_map: number }).owns_map;
         // Integrate only bounded intervals between adjacent samples; outages contribute no invented time.
         const totals = db().prepare(`WITH intervals AS (SELECT *,LEAD(sampled_at) OVER (ORDER BY sampled_at)-sampled_at AS duration
             FROM autoscaling_samples WHERE policy_id=?) SELECT COALESCE(SUM(MIN(duration,15000))/60000.0,0) AS observedMinutes,
             COALESCE(SUM(empty*MIN(duration,15000))/3600000.0,0) AS emptyLobbyHours,
             COALESCE(SUM(CASE WHEN deficit>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS deficitMinutes,
             COALESCE(SUM(CASE WHEN unknown>0 THEN MIN(duration,15000) ELSE 0 END)/60000.0,0) AS unknownMinutes FROM intervals WHERE duration<=30000`).get(policy.id) as ScalingView['totals'];
-        return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals };
+        const operations = db().prepare(`SELECT COALESCE(SUM(d.action='create'),0) AS creates,COALESCE(SUM(d.action='destroy'),0) AS removals,
+            COALESCE(SUM(d.state='cancelled'),0) AS cancelled,COALESCE(SUM(d.state='failed'),0) AS failed,
+            COUNT(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN 1 END) AS readinessSamples,
+            AVG(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMeanSeconds,
+            MAX(CASE WHEN d.action='create' AND d.state IN ('done','warning') THEN (julianday(d.finished_at)-julianday(d.started_at))*86400 END) AS readinessMaxSeconds
+            FROM server_deployments d JOIN provisioning_sources s ON s.operation_id=d.id
+            WHERE s.kind='autoscaling' AND s.source_id=? AND d.started_at>=?`).get(policy.id, new Date(Date.now() - 31 * 86400_000).toISOString()) as ScalingView['operations'];
+        return { policy, snapshot, decision, token: '', blockers: [], history, lastEvaluatedAt: history[0]?.lastSeenAt ?? null, totals, operations, unsettled, ownsMap };
     });
     return { policies, globalPaused: scalingGloballyPaused(), liveAvailable: true, removalAvailable: true };
 }

src/lib/server/autoscaling/schema.ts

@@ -22,6 +22,7 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
         );
         CREATE UNIQUE INDEX IF NOT EXISTS autoscaling_one_intent ON autoscaling_decisions(policy_id) WHERE resolved=0;
         CREATE INDEX IF NOT EXISTS autoscaling_history ON autoscaling_decisions(policy_id,evaluated_at DESC);
+        CREATE INDEX IF NOT EXISTS autoscaling_decision_retention ON autoscaling_decisions(last_seen_at) WHERE resolved=1;
         CREATE TABLE IF NOT EXISTS autoscaling_samples (
             policy_id TEXT NOT NULL REFERENCES autoscaling_policies(id), sampled_at INTEGER NOT NULL,
             mode TEXT NOT NULL, occupied INTEGER NOT NULL, empty INTEGER NOT NULL, unknown INTEGER NOT NULL,
@@ -31,5 +32,6 @@ export function ensureAutoscalingSchema(handle: Database.Database): void {
             operation_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL,
             expires_at INTEGER NOT NULL, claimed_at INTEGER, revoked_at INTEGER
         );
+        CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at);
     `);
 }

src/lib/server/autoscaling/snapshot.ts

@@ -31,8 +31,8 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
     const matches = (value: Parameters<typeof mapIdentity>[0]) => { const identity = mapIdentity(value); return identity?.game === policy.game && identity.mapSlug === policy.mapSlug; };
     const blockers: string[] = [];
     for (const row of rows) if (!mapIdentity(row) && (row.game ?? parseServerName(row.name)?.game) === policy.game) blockers.push(`Resolve ambiguous map identity for ${row.name}.`);
-    const operations = handle.prepare(`SELECT id,server,action,state FROM server_deployments WHERE server_deleted_at IS NULL
-        AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string }[];
+    const operations = handle.prepare(`SELECT id,server,action,state,stage FROM server_deployments WHERE server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=? ORDER BY rowid DESC`).all(exceptOperation) as { id: string; server: string; action: string; state: string; stage: string }[];
     const reservations = handle.prepare(`SELECT j.server AS name,substr(j.server,1,2) AS game,j.map_slug,j.expected_map AS map FROM server_creation_jobs j
         JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND d.id!=?`).all(exceptOperation) as { name: string; game: string; map_slug: string; map: string }[];
     const members: ScalingMember[] = rows.filter(matches).map((row: ServerRow) => {
@@ -50,14 +50,17 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO
             players = row.last_players; observedAt = Date.parse(row.last_seen_at!); source = 'rcon';
         }
         if (protection || !row.native_instance_id || host?.mode !== 'active') { state = 'protected'; blocker = protection ?? 'Native instance on an active host required'; }
-        else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id} is ${operation.state}`; }
+        else if (operation) { state = operation.action === 'create' ? 'creating' : operation.action === 'destroy' ? 'removing' : 'protected'; blocker = `Provisioning ${operation.id}: ${operation.stage}`; }
         else if (row.desired_state === 'stopped' || ['stopped','exited','created','dead'].includes(row.container_state ?? '')) state = 'stopped';
         else if (observation.health === 'online' && players !== null && observedAt !== null && observedAt <= now && freshStatusTimestamp(new Date(observedAt).toISOString(), now)) state = players > 0 ? 'occupied' : 'empty';
         const transitions = [row.inactive_since, observation.inactive_since].map(at => Date.parse(at ?? '')).filter(at => Number.isFinite(at) && at <= now);
         return { name: row.name, identity, state, players, observedAt, source: source ?? null, emptySince: null,
             emptyStartedAt: transitions.length ? Math.max(...transitions) : null, blocker, operationId: operation?.id ?? null };
     });
-    for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: 'Creation is pending.', operationId: operations.find(o => o.server === row.name)?.id ?? null });
+    for (const row of reservations.filter(matches)) if (!members.some(m => m.name === row.name)) {
+        const operation = operations.find(o => o.server === row.name);
+        members.push({ name: row.name, identity: 'reserved', state: 'creating', players: null, observedAt: null, source: null, emptySince: null, blocker: operation?.stage ?? 'Creation is pending.', operationId: operation?.id ?? null });
+    }
     const retained = handle.prepare('SELECT server,observation FROM autoscaling_members WHERE policy_id=?').all(policy.id) as { server: string; observation: string }[];
     for (const member of members) {
         const saved = retained.find(r => r.server === member.name);

src/lib/server/provisioning/native.ts

@@ -500,6 +500,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
+            if (job.action === 'destroy' && scalingIntent(job.id) && (operation.result as { autoscalingInvalidated?: boolean } | null)?.autoscalingInvalidated === true) {
+                db().transaction(() => {
+                    db().prepare("UPDATE server_deployments SET state='cancelled',stage='Automatic removal withdrawn; new empty interval required',finished_at=?,error=NULL WHERE id=?")
+                        .run(new Date().toISOString(), job.id);
+                    recordProgress(job.id, { blocker: null, event: { kind: 'outcome', message: 'Automatic removal was invalidated before mutation. A new decision and empty interval are required.', details: { state: 'cancelled' } } });
+                })();
+                return;
+            }
             if (migration) {
                 if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
                     db().transaction(() => {

src/routes/autoscaling/+page.svelte

@@ -59,7 +59,7 @@
 <svelte:head><title>Autoscaling | Electric Cherry</title></svelte:head>
 <div class="page">
     <SectionHeader title="Autoscaling" subtitle="Keep empty lobbies available for new groups, within the limits you choose.">
-        {#snippet actions()}<Button onclick={() => refresh()}>Refresh</Button>{#if canOperate}<Button disabled={busy} onclick={pauseAll}>{report.globalPaused ? 'Resume autoscaling' : 'Pause all'}</Button><Button variant="primary" onclick={() => edit(null)}>Add policy</Button>{/if}{/snippet}
+        {#snippet actions()}<Button onclick={() => refresh()}>Refresh</Button>{#if canOperate}<Button disabled={busy} onclick={pauseAll}>{report.globalPaused ? 'Resume autoscaling' : 'Pause all'}</Button><Button variant="primary" disabled={busy} onclick={() => edit(null)}>Add policy</Button>{/if}{/snippet}
     </SectionHeader>
     <p class="context"><a href="/">Fleet</a> · <a href="/collections">Collections</a> · <a href="/metrics">Metrics</a></p>
     {#if report.globalPaused}<p class="banner">All autoscaling decisions are paused. Delivered operations may still need to settle.</p>{/if}
@@ -101,8 +101,9 @@
     {#if !report.policies.length}<p class="panel">No autoscaling policies. Add a policy to observe a map in dry run.</p>{/if}
     {#each report.policies as view (view.policy.id)}
         <section class="panel" id={view.policy.id} aria-labelledby={`title-${view.policy.id}`}>
-            <div class="heading"><h2 id={`title-${view.policy.id}`}>{zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}</h2><span>{modeLabel(view.policy.mode)}</span>{#if canOperate}<Button onclick={() => edit(view.policy)}>Edit or pause</Button>{/if}</div>
+            <div class="heading"><h2 id={`title-${view.policy.id}`}>{zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}</h2><span>{modeLabel(view.policy.mode)}</span>{#if canOperate}<Button disabled={busy} onclick={() => edit(view.policy)}>Edit or pause</Button>{/if}</div>
             <p class="counts">{view.decision.occupied} occupied · {view.decision.empty} empty and ready · {view.decision.pending} pending · {view.decision.unknown} unknown or stopped</p>
+            {#if view.unsettled.length}<div class="banner"><strong>Work is still settling.</strong>{#each view.unsettled as operation}<p><a href={`/provisioning/${operation.id}`}>{operation.action} · {operation.state}</a> — {operation.stage}</p>{/each}{#if view.ownsMap}<p>This policy retains map ownership until its outstanding work settles.</p>{/if}</div>{/if}
             <p>Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyedNames.length} saved keys</p>
             <p><strong>{view.policy.mode === 'dry-run' ? 'Hypothetical decision: ' : 'Current decision: '}</strong>{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}</p>
             {#if view.policy.mode === 'create-only' && view.decision.action === 'destroy'}<p>Removal is advisory in Create only mode.</p>{/if}
@@ -110,8 +111,10 @@
             <details><summary>Lobbies and evidence</summary><ul>{#each view.snapshot.members as member}<li><a href={`/servers/${encodeURIComponent(member.name)}`}>{member.name}</a> · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}<br />{member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · <a href={`/provisioning/${member.operationId}`}>Provisioning</a>{/if}</li>{/each}</ul>{#if !view.snapshot.members.length}<p>No registered or pending lobbies.</p>{/if}{#each view.snapshot.blockers as blocker}<p>{blocker}</p>{/each}</details>
             <details><summary>Recent decisions and measurements</summary>
                 <p>{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes</p>
+                <p>Last 31 days: {view.operations.creates} create requests · {view.operations.removals} removal requests · {view.operations.cancelled} cancelled · {view.operations.failed} failed</p>
+                {#if view.operations.readinessSamples}<p>Request to verified readiness: mean {view.operations.readinessMeanSeconds?.toFixed(0)} seconds, maximum {view.operations.readinessMaxSeconds?.toFixed(0)} seconds across {view.operations.readinessSamples} completed creates. Includes queue time.</p>{/if}
                 <p class="muted">Up to 31 days of bounded observations; gaps are excluded. Dry run measures the actual fleet, not a simulated replacement lineup. Readiness does not certify browser visibility or successful joins.</p>
-                <ol>{#each view.history as item}<li>{$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}<br />{item.decision.reason}{#if item.operationId} · <a href={`/provisioning/${item.operationId}`}>View operation</a>{/if}</li>{/each}</ol>
+                <ol>{#each view.history as item}<li>{$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}<br />{item.decision.reason}{#if item.operationId} · <a href={`/provisioning/${item.operationId}`}>{item.operationState ?? 'View operation'}</a>{/if}</li>{/each}</ol>
             </details>
         </section>
     {/each}
@@ -122,7 +125,7 @@
     .panel { margin: 20px 0; padding: 20px; border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); background: var(--surface); overflow-wrap: anywhere; }
     .heading, .actions { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; }
     h2 { font-size: var(--text-lg); margin: 0; } h3 { font-size: var(--text-md); }
-    .heading h2 { flex: 1; } p, li { line-height: 1.6; } .muted, .context { color: var(--text-secondary); font-size: var(--text-sm); }
+    .heading h2 { flex: 1 1 200px; } p, li { line-height: 1.6; } .muted, .context { color: var(--text-secondary); font-size: var(--text-sm); }
     .fields { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 220px), 1fr)); gap: 16px; margin-bottom: 20px; }
     label { display: flex; flex-direction: column; gap: 8px; font-size: var(--text-sm); }
     input, select { padding: 10px; min-width: 0; color: var(--text-primary); background: var(--surface); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); font: inherit; }