Add autoscaling outcomes to Metrics (Refs #140)

tests/unit/database-migrations.test.ts

@@ -205,3 +205,41 @@ it('adds match report invalidation on upgrade without invalidating for score hea
     expect(revision()).toBeGreaterThan(renamed);
     expect(upgraded.prepare('SELECT * FROM leaderboard_match_players').all()).toEqual([]);
 });
+
+it('adds provisioning report invalidation at version 14 without invalidating for progress updates', async () => {
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    for (const table of ['server_deployments', 'provisioning_sources'])
+        for (const action of ['INSERT', 'UPDATE', 'DELETE'])
+            handle.exec(`DROP TRIGGER report_metrics_${table}_${action}`);
+    handle.pragma('user_version = 14');
+    closeDb();
+    const upgraded = db();
+    const revision = () =>
+        (
+            upgraded
+                .prepare("SELECT revision FROM report_revisions WHERE topic='metrics'")
+                .get() as { revision: number }
+        ).revision;
+    const before = revision();
+    upgraded.exec(`INSERT INTO server_deployments
+        (id,server,action,state,stage,started_at,triggered_by)
+        VALUES('operation','t5_zombie_moon_1','create','waiting','Queued',
+            '2026-10-10T12:00:00.000Z','test')`);
+    expect(revision()).toBe(before);
+    upgraded.exec(`INSERT INTO provisioning_sources(operation_id,kind,source_id,server)
+        VALUES('operation','autoscaling','policy','t5_zombie_moon_1')`);
+    expect(revision()).toBeGreaterThan(before);
+    const linked = revision();
+    upgraded.exec(
+        "UPDATE server_deployments SET stage='Still waiting',observed_at='2026-10-10T12:00:15.000Z'"
+    );
+    expect(revision()).toBe(linked);
+    upgraded.exec(
+        "UPDATE server_deployments SET state='done',finished_at='2026-10-10T12:01:12.000Z'"
+    );
+    expect(revision()).toBeGreaterThan(linked);
+    const completed = revision();
+    upgraded.exec('DELETE FROM server_deployments');
+    expect(revision()).toBeGreaterThan(completed);
+});

tests/unit/metrics-autoscaling.test.ts

@@ -0,0 +1,189 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { parseMetricsQuery } from '../../src/lib/metrics.js';
+
+let fixture: Fixture;
+const now = Date.UTC(2026, 9, 10, 12);
+const day = 86400_000;
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+});
+afterEach(async () => {
+    vi.restoreAllMocks();
+    await fixture.cleanup();
+});
+
+async function operation(
+    id: string,
+    options: {
+        server?: string;
+        action?: string;
+        state?: string;
+        start?: number | string;
+        finish?: number | null;
+        source?: string;
+    } = {}
+) {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    const server = options.server ?? 't5_zombie_moon_1';
+    const finish = options.finish === undefined ? now - 1000 : options.finish;
+    const start = options.start ?? (finish ?? now) - 72_000;
+    handle
+        .prepare(
+            `INSERT INTO server_deployments
+        (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by)
+        VALUES(?,?,? ,?,?,'test',?,?,'test')`
+        )
+        .run(
+            id,
+            server,
+            new Date(now).toISOString(),
+            options.action ?? 'create',
+            options.state ?? 'done',
+            typeof start === 'string' ? start : new Date(start).toISOString(),
+            finish === null ? null : new Date(finish).toISOString()
+        );
+    handle
+        .prepare(
+            `INSERT INTO provisioning_sources(operation_id,kind,source_id,server)
+        VALUES(?,?, 'deleted-policy',?)`
+        )
+        .run(id, options.source ?? 'autoscaling', server);
+    return handle;
+}
+
+it.each([1, 7, 30] as const)(
+    'counts outcomes by finish time in the inclusive %i-day window',
+    async (days) => {
+        const { readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
+        const since = now - days * day;
+        await operation('before', { finish: since - 1 });
+        await operation('at-start', { finish: since, start: since - 60_000 });
+        await operation('at-end', { finish: now, start: now - 120_000, state: 'warning' });
+        await operation('after', { finish: now + 1 });
+        await operation('removed', { action: 'destroy' });
+        await operation('removed-warning', { action: 'destroy', state: 'warning' });
+        await operation('failed-create', { state: 'failed' });
+        await operation('failed-removal', { action: 'destroy', state: 'failed' });
+        await operation('cancelled', { state: 'cancelled' });
+        await operation('pending', { state: 'awaiting-health', finish: null });
+        await operation('interrupted', { state: 'interrupted' });
+        await operation('manual', { source: 'maintenance' });
+        await operation('rebuild', { action: 'rebuild' });
+        const result = readMetrics(days, now);
+        expect(result.autoscaling).toEqual({
+            added: 2,
+            removed: 2,
+            failed: 2,
+            cancelled: 1,
+            readinessSamples: 2,
+            readinessMeanSeconds: 90,
+            readinessMaxSeconds: 120
+        });
+        expect(result.summary.observed).toBe(0);
+        expect(result.maps).toEqual([]);
+        expect(result.scopeOptions.maps).toEqual([{ game: 't5', map: 'Moon' }]);
+    }
+);
+
+it('weights readiness by completed creates and counts each operation once after policy/server deletion', async () => {
+    const { readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
+    const handle = await operation('one', { start: now - 61_000 });
+    await operation('two', { start: now - 121_000 });
+    await operation('three', { server: 't6_zombie_town_1', start: now - 181_000 });
+    // Several source rows and another consumer do not multiply the result.
+    handle
+        .prepare(
+            `INSERT INTO provisioning_sources(operation_id,kind,source_id,server)
+        VALUES('one','autoscaling','second-source','t5_zombie_moon_1'),
+              ('one','maintenance','maintenance-source','t5_zombie_moon_1')`
+        )
+        .run();
+    expect(handle.prepare('SELECT * FROM autoscaling_policies').all()).toEqual([]);
+    expect(handle.prepare('SELECT * FROM autoscaling_decisions').all()).toEqual([]);
+    expect(handle.prepare('SELECT * FROM servers').all()).toEqual([]);
+    expect(readMetrics(7, now).autoscaling).toEqual({
+        added: 3,
+        removed: 0,
+        failed: 0,
+        cancelled: 0,
+        readinessSamples: 3,
+        readinessMeanSeconds: 120,
+        readinessMaxSeconds: 180
+    });
+});
+
+it('filters operation identity by game and map variant independently of the policy scope', async () => {
+    const { readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
+    await operation('moon');
+    await operation('town', { server: 't6_zombie_town_1', action: 'destroy' });
+    await operation('farm', { server: 't6_zombie_farm_1', state: 'failed' });
+    await operation('transit', { server: 't6_zombie_transit_1', state: 'cancelled' });
+    await operation('older', { server: 't5_zombie_theater_1', finish: now - 8 * day });
+    await operation('future', { server: 't6_zombie_tomb_1', finish: now + 1 });
+    const report = (game: string, map = '') =>
+        readMetrics(parseMetricsQuery(new URLSearchParams({ days: '7', game, map })), now);
+    expect(report('t6').autoscaling).toMatchObject({
+        added: 0,
+        removed: 1,
+        failed: 1,
+        cancelled: 1
+    });
+    expect(report('t5', 'Moon').autoscaling.added).toBe(1);
+    expect(report('t5', 'zombie_moon').autoscaling.added).toBe(1);
+    for (const label of ['Town Survival', 'town']) {
+        expect(report('t6', label).autoscaling).toMatchObject({
+            added: 0,
+            removed: 1,
+            failed: 0,
+            cancelled: 0
+        });
+    }
+    for (const [game, map] of [
+        ['t6', 'zm_transit'],
+        ['t5', 'Town Survival'],
+        ['t6', 'Unknown']
+    ])
+        expect(report(game, map).autoscaling).toMatchObject({
+            added: 0,
+            removed: 0,
+            failed: 0,
+            cancelled: 0
+        });
+    const options = report('t6').scopeOptions;
+    expect(options.maps).toContainEqual({ game: 't5', map: 'Kino Der Toten' });
+    expect(options.maps).not.toContainEqual({ game: 't6', map: 'Origins' });
+    expect(report('t5', 'Kino Der Toten').autoscaling.added).toBe(0);
+    expect(readMetrics(30, now).autoscaling.added).toBe(2);
+});
+
+it('keeps absent readiness null and excludes invalid durations without losing completed counts', async () => {
+    const { readMetrics } = await import('../../src/lib/server/fleet/metrics.js');
+    expect(readMetrics(7, now).autoscaling).toEqual({
+        added: 0,
+        removed: 0,
+        failed: 0,
+        cancelled: 0,
+        readinessSamples: 0,
+        readinessMeanSeconds: null,
+        readinessMaxSeconds: null
+    });
+    await operation('negative', { start: now });
+    await operation('invalid-start', { start: 'not-recorded' });
+    await operation('missing-finish', { finish: null });
+    expect(readMetrics(7, now).autoscaling).toMatchObject({
+        added: 2,
+        readinessSamples: 0,
+        readinessMeanSeconds: null,
+        readinessMaxSeconds: null
+    });
+    await operation('zero', { start: now, finish: now });
+    expect(readMetrics(7, now).autoscaling).toMatchObject({
+        added: 3,
+        readinessSamples: 1,
+        readinessMeanSeconds: 0,
+        readinessMaxSeconds: 0
+    });
+});

tests/unit/metrics-refresh.test.ts

@@ -17,6 +17,15 @@ const report = (days: number) => ({
     summary: totals,
     running: { average: null, peak: 0, capturedHours: 0 },
     matches: emptyMatches,
+    autoscaling: {
+        added: 0,
+        removed: 0,
+        failed: 0,
+        cancelled: 0,
+        readinessSamples: 0,
+        readinessMeanSeconds: null,
+        readinessMaxSeconds: null
+    },
     points: [],
     maps: [],
     sampledServerCount: 0,
@@ -71,6 +80,26 @@ function setup(fetcher: typeof fetch) {
 }
 
 describe('metrics background refresh', () => {
+    it.each([
+        null,
+        { ...report(7).autoscaling, added: -1 },
+        { ...report(7).autoscaling, cancelled: 1.5 },
+        { ...report(7).autoscaling, readinessMeanSeconds: '72' },
+        { ...report(7).autoscaling, readinessMaxSeconds: -1 }
+    ])(
+        'keeps the previous report when autoscaling measurements are invalid (%j)',
+        async (autoscaling) => {
+            const refresh = setup(
+                vi.fn().mockResolvedValue(Response.json({ ...report(7), autoscaling }))
+            );
+            await refresh.refresh();
+            expect(refresh.onData).not.toHaveBeenCalled();
+            expect(refresh.onError).toHaveBeenLastCalledWith(
+                'The metrics response was incomplete.'
+            );
+        }
+    );
+
     it.each([undefined, null, -1, 1.5, '1'])(
         'rejects an invalid sampled-server count (%s) and accepts a count without server rows',
         async (sampledServerCount) => {

tests/unit/report-workers.test.ts

@@ -158,3 +158,45 @@ it('refreshes policy signals after policy edits and global pauses without new sa
     db().prepare("DELETE FROM autoscaling_policies WHERE id='moon-policy'").run();
     expect((await metricsReport(query)).maps[0].policy).toBeNull();
 });
+
+it('refreshes autoscaling outcomes after completion, source changes and retention without new samples', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { metricsReport } = await import('../../src/lib/server/reports/index.js');
+    const now = Date.UTC(2026, 9, 10, 12);
+    vi.spyOn(Date, 'now').mockReturnValue(now);
+    const query = parseMetricsQuery(new URLSearchParams('days=7'));
+    const report = () => metricsReport(query);
+    expect((await report()).autoscaling.added).toBe(0);
+    db()
+        .prepare(
+            `INSERT INTO server_deployments
+        (id,server,action,state,stage,started_at,triggered_by)
+        VALUES('operation','t5_zombie_moon_1','create','awaiting-health','Waiting',?,'test')`
+        )
+        .run(new Date(now - 72_000).toISOString());
+    db().exec(`INSERT INTO provisioning_sources(operation_id,kind,source_id,server)
+        VALUES('operation','autoscaling','deleted-policy','t5_zombie_moon_1')`);
+    expect((await report()).autoscaling.added).toBe(0);
+    db()
+        .prepare("UPDATE server_deployments SET state='done',finished_at=? WHERE id='operation'")
+        .run(new Date(now).toISOString());
+    expect((await report()).autoscaling).toMatchObject({
+        added: 1,
+        readinessSamples: 1,
+        readinessMeanSeconds: 72,
+        readinessMaxSeconds: 72
+    });
+    db().exec("UPDATE provisioning_sources SET kind='maintenance'");
+    expect((await report()).autoscaling.added).toBe(0);
+    db().exec("UPDATE provisioning_sources SET kind='autoscaling'");
+    expect((await report()).autoscaling.added).toBe(1);
+    db().exec("UPDATE server_deployments SET state='failed'");
+    expect((await report()).autoscaling).toMatchObject({
+        added: 0,
+        failed: 1,
+        readinessSamples: 0
+    });
+    db().exec('DELETE FROM server_deployments');
+    expect((await report()).autoscaling.failed).toBe(0);
+    expect((await report()).scopeOptions.maps).toEqual([]);
+});

tests/unit/ui-metrics-page.test.ts

@@ -3,6 +3,7 @@ import { render } from 'svelte/server';
 import type { MetricsData, MapMetrics, MatchMetrics } from '../../src/lib/metrics.js';
 import MetricsPage from '../../src/routes/metrics/+page.svelte';
 import MetricsSummary from '../../src/lib/ui/components/metrics/MetricsSummary.svelte';
+import MetricsAutoscaling from '../../src/lib/ui/components/metrics/MetricsAutoscaling.svelte';
 import MetricsTables from '../../src/lib/ui/components/metrics/MetricsTables.svelte';
 import MetricsExplore from '../../src/lib/ui/components/metrics/MetricsExplore.svelte';
 import MetricsChart from '../../src/lib/ui/components/metrics/MetricsChart.svelte';
@@ -64,6 +65,15 @@ function report(): MetricsData {
         scopeFirstSampleAt: now - 14 * 86400000,
         scopeLatestSampleAt: now,
         matches: matches(),
+        autoscaling: {
+            added: 73,
+            removed: 48,
+            failed: 1,
+            cancelled: 2,
+            readinessSamples: 73,
+            readinessMeanSeconds: 72,
+            readinessMaxSeconds: 168
+        },
         summary,
         running: { average: 12, peak: 24, capturedHours: 1 },
         maps: Array.from({ length: 24 }, (_, i) => map(i + 1)),
@@ -106,6 +116,50 @@ function scope(search: string, stateSearch?: string) {
 }
 beforeEach(() => scope(''));
 
+it('shows Provisioning outcomes and readiness with a policy link, even without occupancy', () => {
+    const data = report();
+    const section = render(MetricsAutoscaling, { props: { metrics: data.autoscaling } }).body;
+    for (const label of [
+        'Lobbies added',
+        'Lobbies removed',
+        'Failed operations',
+        'Cancelled operations'
+    ])
+        expect(section).toContain(label);
+    for (const value of [73, 48, 1, 2]) expect(section).toContain(`>${value}<`);
+    expect(section).toContain('72 s');
+    expect(section).toContain('168 s');
+    expect(section).toContain('href="/autoscaling"');
+    expect(section.replace(/\s+/g, ' ')).toContain('73 completed creates with recorded timing');
+    data.summary = { observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 };
+    data.latestSampleAt = null;
+    const body = render(MetricsPage, {
+        props: { data: { ...data, user: null, setupRequired: false } }
+    }).body;
+    expect(body).toContain('No retained observations yet');
+    expect(body).toContain('Lobbies added');
+    expect(body).toContain('72 s');
+});
+
+it('distinguishes no autoscaling outcomes from unavailable readiness', () => {
+    const body = render(MetricsAutoscaling, {
+        props: {
+            metrics: {
+                added: 0,
+                removed: 0,
+                failed: 0,
+                cancelled: 0,
+                readinessSamples: 0,
+                readinessMeanSeconds: null,
+                readinessMaxSeconds: null
+            }
+        }
+    }).body.replace(/\s+/g, ' ');
+    expect(body).toContain('No completed, failed or cancelled autoscaling operations recorded');
+    expect(body).toContain('Request-to-ready time is unavailable');
+    expect(body).not.toContain('0 s');
+});
+
 it.each([
     [0, 'No captured time in this scope'],
     [1, 'Waiting for observed time']