From eadf97a4286cbac4bb5e9061f39c6be22b5cd82f Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Wed, 7 Oct 2026 15:49:46 -0400 Subject: [PATCH] Remove dry-run and create-only autoscaling modes --- docs/README.md | 2 +- docs/guides/autoscaling.md | 21 ++-- docs/guides/getting-started.md | 2 +- docs/plans/lobby-autoscaling-plan.md | 4 +- ...er-achievements-and-visual-refresh-plan.md | 2 +- src/lib/autoscaling.ts | 4 +- src/lib/server/autoscaling/admission.ts | 1 - src/lib/server/autoscaling/index.ts | 2 +- src/lib/server/autoscaling/schema.ts | 12 ++ src/routes/autoscaling/+page.svelte | 15 ++- tests/unit/autoscaling-delete.test.ts | 12 +- tests/unit/autoscaling-dry-run.test.ts | 62 ---------- tests/unit/autoscaling-game.test.ts | 2 +- tests/unit/autoscaling-keys.test.ts | 9 +- tests/unit/autoscaling-live.test.ts | 7 +- tests/unit/autoscaling-planner.test.ts | 2 +- tests/unit/autoscaling-policies.test.ts | 116 ++++++++++++++++++ tests/unit/autoscaling-snapshot.test.ts | 2 +- 18 files changed, 172 insertions(+), 105 deletions(-) delete mode 100644 tests/unit/autoscaling-dry-run.test.ts create mode 100644 tests/unit/autoscaling-policies.test.ts diff --git a/docs/README.md b/docs/README.md index f7f567e..83c674b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,7 +38,7 @@ proposes calendar leaderboards, personal-best highlights, shareable result cards and consistent map and status presentation. [Lobby autoscaling](plans/lobby-autoscaling-plan.md) records the implemented design -and the production dry run, pilot measurements, and rollout still to complete. +and the production pilot measurements and rollout still to complete. Current operating instructions live in the guides above. ## Research diff --git a/docs/guides/autoscaling.md b/docs/guides/autoscaling.md index 107b101..e08969b 100644 --- a/docs/guides/autoscaling.md +++ b/docs/guides/autoscaling.md @@ -2,30 +2,30 @@ Open **Fleet → Autoscaling** to keep empty lobbies available for new groups. Policies apply to one map variant or every map in a game. A game-wide policy applies the same minimum, maximum, and spare targets independently to each map, including all variants. A partly occupied Zombies lobby is not an empty spare, even when it still has player slots. -Autoscaling is opt-in. Installing the feature creates no policies and changes no lobby counts. Use a two-week dry run before enabling live actions. +Autoscaling is opt-in. Installing the feature creates no policies and changes no lobby counts. Review the activation preview before enabling Create and destroy. T5 game-wide policies cover the ten standard Zombies maps. Dead Ops Arcade is not supported. ## Configure a policy -Choose **Add policy**, select the game and scope, and enter an explicit maximum. The initial editor selects Dry run, with a minimum of one running lobby, one empty spare, a 20-minute empty interval, and a five-minute action cooldown. Counts must be between 1 and 20; the minimum and spare target cannot exceed the maximum. +Choose **Add policy**, select the game and scope, and enter an explicit maximum. The initial editor selects Off, with a minimum of one running lobby, one empty spare, a 20-minute empty interval, and a five-minute action cooldown. Counts must be between 1 and 20; the minimum and spare target cannot exceed the maximum. Choose **Preview changes** to inspect the existing lobbies, game key availability, target, proposed action, and blockers. **Save reviewed policy** checks the policy revision and current fleet again. If the preview changed, review a new preview before saving. | Mode | Behavior | | --- | --- | | Off | No new actions. Unsubmitted work is withdrawn and delivered operations remain tracked until settled. | -| Dry run | Records hypothetical decisions without reserving identities, hosts, or ports. Collections retain control of the lineup. | -| Create only | Adds a lobby when eligible. Surplus removal remains advisory. | | Create and destroy | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. | Plutonium keys belong to a game pool shared by every map, manual launch, collection, beta, and autoscaling policy for that game. An eligible creation uses the lowest available lobby number from 1–20 and reserves any free game key atomically. Key labels do not select a map or lobby. The preview shows saved and available game keys; policy maxima can exceed pool size, and a policy can be saved before any keys are added. An empty or exhausted pool holds new growth with a clear reason; it does not prevent surplus removal. Queued launches reserve their key, stopped instances retain it, and confirmed cleanup returns it to the pool. Failed or uncertain launches keep their reservation until safely reconciled. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host. The desired count is `min(maximum, max(minimum, occupied lobbies + empty spare target))`. The controller evaluates policies every 15 seconds and requests at most one unresolved operation per policy. Game-wide policies share one cooldown and one outstanding operation across their maps, prioritizing additions before removals. Create uses the existing startup concurrency, CPU, memory, library, port, and host admission checks. Lack of host capacity leaves one request waiting; it does not create additional requests every tick. Creates use current promoted launch files when provisioning begins. +Existing policies saved in the removed Dry run or Create only modes switch to Off on upgrade. Review and enable Create and destroy to resume; pending work remains tracked until settled. + ## Ownership and blockers -A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected; dry-run policies may overlap. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. Competing collection launches and manual Create, Start, Stop, Rebuild, or Destroy requests are rejected with a policy link. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish. +A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. Competing collection launches and manual Create, Start, Stop, Rebuild, or Destroy requests are rejected with a policy link. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish. Unknown or stale occupancy, stopped or unhealthy members, active betas, campaigns, maintenance, host drains, and unresolved recovery hold the policy's decisions. Autoscaling does not repair these conditions itself. The page and linked Provisioning operation explain the current blocker, including capacity and game key waits. @@ -49,15 +49,14 @@ Choose **Delete policy** and confirm to remove its settings, decisions, and meas Deletion requires the current policy revision and no unsettled autoscaling operation. If work is outstanding, turn the policy Off and wait for its operation to settle before deleting. Delivered or uncertain host commands retain their policy and admission evidence until reconciliation finishes. -## Dry run and pilot +## Pilot The production observation period and live pilot have not been run as part of implementation. Begin them on the operator-selected maps after deploying the controller and compatible agents. -1. Select one or two representative maps and save policies in Dry run for two weeks. Record the baseline lineup, keys, versions, host CPU and memory, player-hours, and comparable days of the week. -2. Review recent hypothetical actions, empty lobby-hours, minutes below the spare target, unknown-status minutes, capacity/key blockers, and projected action frequency. Dry run observes the actual lineup; it cannot predict how players would redistribute after a removal. -3. Enable Create only on the pilot maps after resolving blockers. Measure request-to-ready time from Provisioning, browser visibility, and successful external joins separately. Increase the spare target if creation latency leaves new groups without a suitable lobby. -4. Agree the acceptable spare-deficit duration for the pilot, verify agent capability, and enable Create and destroy. Keep normal gameplay and joins under observation while comparing actual host CPU and empty lobby-hours with the baseline. -5. Expand one map at a time only when savings are measurable and spare availability and gameplay meet the agreed tolerance. Pause immediately if those conditions regress. +1. Select one or two representative maps and record the baseline lineup, keys, versions, host CPU and memory, and player-hours. +2. Preview Create and destroy after resolving blockers and checking agent capability. Review the affected lobbies, proposed action, minimum, maximum, and spare target before saving. +3. Monitor request-to-ready time from Provisioning, browser visibility, successful external joins, empty lobby-hours, spare deficits, and removals. Increase the spare target if creation latency leaves new groups without a suitable lobby. +4. Expand one map at a time when spare availability and gameplay meet your requirements. Pause if they regress. The page retains up to 31 days of decision evidence and observations, with the 30 most recent decision changes shown. Repeated unchanged evaluations update the decision's last-seen time instead of adding a row. Unresolved intent survives retention. Provisioning history retains the operation and policy backlink separately. diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index e12f4fd..bda6bb0 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -105,4 +105,4 @@ published game files. Preserve that directory across upgrades. Review [Fleet settings](settings.md) for concurrency and retention controls. For lobby autoscaling, start with the documented -[dry run and pilot](autoscaling.md#dry-run-and-pilot) before enabling live actions. +[pilot procedure](autoscaling.md#pilot) before enabling live actions. diff --git a/docs/plans/lobby-autoscaling-plan.md b/docs/plans/lobby-autoscaling-plan.md index cda850b..f36f74b 100644 --- a/docs/plans/lobby-autoscaling-plan.md +++ b/docs/plans/lobby-autoscaling-plan.md @@ -1,6 +1,6 @@ # Lobby autoscaling implementation plan -Status: Approved and implemented October 6, 2026. Controller support and agent 1.7.0 are complete; production dry run, pilot measurements, and live rollout remain pending. See [usage and the pilot procedure](../guides/autoscaling.md). +Status: Approved and implemented October 6, 2026. Controller support and agent 1.7.0 are complete; production pilot measurements and live rollout remain pending. See [usage and the pilot procedure](../guides/autoscaling.md). The design below preserves the approved scope and subsequent implementation record. Use the linked guide for current policy behavior, including game-wide @@ -10,6 +10,8 @@ Introduce an opt-in policy for each game and map variant that keeps a small numb Start with Create/Destroy on existing native hosts. Run the evaluator in dry run for two weeks, then enable creation for a small pilot before enabling removal. Host acquisition, VM shutdown, Stop/Start pooling, scheduled demand prediction, and waking servers from incoming packets are outside the first release. +Update October 7, 2026: Dry run and Create only have been removed. Current policies support Off and Create and destroy, defaulting to Off. Existing policies in removed modes switch to Off on upgrade. The original implementation stages below record the initial design; follow the current [usage and pilot procedure](../guides/autoscaling.md) for operation. + ## Existing foundations and required extensions | Foundation | Reuse and required work | diff --git a/docs/plans/player-achievements-and-visual-refresh-plan.md b/docs/plans/player-achievements-and-visual-refresh-plan.md index 44e39ca..f8493ef 100644 --- a/docs/plans/player-achievements-and-visual-refresh-plan.md +++ b/docs/plans/player-achievements-and-visual-refresh-plan.md @@ -216,7 +216,7 @@ styling, and document their use. | --- | --- | | Fleet cards | Map/lobby/game first; health, player count, and round next; roster and match duration below; lower-priority timing and technical identity in supporting details. Preserve compact empty cards, search, pins, table mode, and the match spotlight. | | Server detail | Add map/game identity to the shared header with health. Keep player counts, round, and timing in the Live content so Configuration and Diagnostics remain focused on their task. | -| Autoscaling | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Dry run/live mode, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. | +| Autoscaling | Use shared identity and mode badges, a compact count strip, and a prominent current decision/blocker. Keep per-map targets, Off/Create and destroy mode, ownership, global pause, and unsettled operations visible. Reuse existing evidence/history disclosures. | | Provisioning and host operations | Use consistent labels and status components while preserving action, target, result, elapsed time, and specific wait reasons. Put long IDs, source paths, and low-level metadata behind explicit details/copy controls. | | Player and match results | Reuse map identity and achievement treatment across profile highlights, match results, and downloaded cards. Distinguish a current record from a record earned at that time. | diff --git a/src/lib/autoscaling.ts b/src/lib/autoscaling.ts index 619c812..62869e9 100644 --- a/src/lib/autoscaling.ts +++ b/src/lib/autoscaling.ts @@ -1,6 +1,6 @@ import { zombieMapBySlug, type ZombieGame } from './map-aliases.js'; -export const autoscalingModes = ['off', 'dry-run', 'create-only', 'create-destroy'] as const; +export const autoscalingModes = ['off', 'create-destroy'] as const; export type AutoscalingMode = typeof autoscalingModes[number]; export interface ScalingSettings { game: ZombieGame; @@ -35,7 +35,7 @@ export interface ScalingView extends ScalingPreview { unsettled: { id: string; action: string; state: string; stage: string }[]; } export interface ScalingReport { policies: ScalingView[]; globalPaused: boolean; liveAvailable: boolean; removalAvailable: boolean } -export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-only' || mode === 'create-destroy'; +export const liveScaling = (mode: AutoscalingMode): boolean => mode === 'create-destroy'; export function scalingSettings(input: unknown): ScalingSettings { if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide an autoscaling policy.'); diff --git a/src/lib/server/autoscaling/admission.ts b/src/lib/server/autoscaling/admission.ts index b81621b..5c089eb 100644 --- a/src/lib/server/autoscaling/admission.ts +++ b/src/lib/server/autoscaling/admission.ts @@ -19,7 +19,6 @@ export function scalingAdmission(operationId: string): string | null { else { const policy: ScalingPolicy = { ...JSON.parse(row.settings), id: row.id, mode: row.mode, revision: row.revision, nextActionAt: 0, pausedReason: null, updatedAt: '' }; const requested = JSON.parse(intent.decision) as ScalingDecision; - if (requested.action === 'destroy' && row.mode !== 'create-destroy') return 'Automatic removal is disabled.'; const snapshot = scalingSnapshot(policy, Date.now(), operationId); if (requested.action === 'create') { snapshot.reservedNames = snapshot.reservedNames.filter(n => n !== requested.server); diff --git a/src/lib/server/autoscaling/index.ts b/src/lib/server/autoscaling/index.ts index cdc5f75..71fe3a0 100644 --- a/src/lib/server/autoscaling/index.ts +++ b/src/lib/server/autoscaling/index.ts @@ -150,7 +150,7 @@ export async function runAutoscaling(): Promise { const evidence = JSON.stringify({ members: snapshot.members, reservations: snapshot.reservedNames, keyPool: snapshot.keyPool, blockers: snapshot.blockers }); const fingerprint = createHash('sha256').update(JSON.stringify([policy.revision, decision, snapshot.members.map(m => [m.name, m.identity, m.state])])).digest('hex'); const last = db().prepare('SELECT id,fingerprint FROM autoscaling_decisions WHERE policy_id=? ORDER BY evaluated_at DESC,rowid DESC LIMIT 1').get(policy.id) as { id: string; fingerprint: string } | undefined; - const execute = liveScaling(policy.mode) && (decision.action === 'create' || policy.mode === 'create-destroy' && decision.action === 'destroy'); + const execute = liveScaling(policy.mode) && (decision.action === 'create' || decision.action === 'destroy'); const decisionId = randomUUID(); if (!execute && last?.fingerprint === fingerprint) db().prepare('UPDATE autoscaling_decisions SET last_seen_at=?,evidence=? WHERE id=?').run(now, evidence, last.id); else db().prepare('INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence) VALUES(?,?,?,?,?,?,?,?)') diff --git a/src/lib/server/autoscaling/schema.ts b/src/lib/server/autoscaling/schema.ts index 4bcd01c..6aab73e 100644 --- a/src/lib/server/autoscaling/schema.ts +++ b/src/lib/server/autoscaling/schema.ts @@ -34,4 +34,16 @@ export function ensureAutoscalingSchema(handle: Database.Database): void { ); CREATE INDEX IF NOT EXISTS autoscaling_sample_retention ON autoscaling_samples(sampled_at); `); + // Removed modes must not silently gain permission to destroy lobbies. + // Keep ownership while delivered work settles through normal reconciliation. + handle.transaction(() => { + handle.prepare(`UPDATE autoscaling_authorizations SET revoked_at=? + WHERE claimed_at IS NULL AND revoked_at IS NULL AND policy_id IN + (SELECT id FROM autoscaling_policies WHERE mode IN ('dry-run','create-only'))`).run(Date.now()); + handle.prepare(`UPDATE autoscaling_policies SET mode='off', settings=json_set(settings,'$.mode','off'), + revision=revision+1, paused_reason='This autoscaling mode was removed. Review and enable Create and destroy to resume.', + owns_map=CASE WHEN EXISTS (SELECT 1 FROM autoscaling_decisions WHERE policy_id=autoscaling_policies.id AND resolved=0) + THEN owns_map ELSE 0 END, updated_at=? + WHERE mode IN ('dry-run','create-only')`).run(new Date().toISOString()); + })(); } diff --git a/src/routes/autoscaling/+page.svelte b/src/routes/autoscaling/+page.svelte index 68d1143..a257866 100644 --- a/src/routes/autoscaling/+page.svelte +++ b/src/routes/autoscaling/+page.svelte @@ -12,14 +12,14 @@ let snapshot = $state(null); const report = $derived(snapshot ?? data.autoscaling); const canOperate = $derived(page.data.user?.role === 'admin' || page.data.user?.role === 'operator'); - const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'dry-run', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20, cooldownMinutes: 5 }); + const defaults = (): ScalingSettings => ({ game: 't5', mapSlug: 'theater', mode: 'off', minimum: 1, maximum: 4, spares: 1, emptyMinutes: 20, cooldownMinutes: 5 }); let settings = $state(defaults()); let editing = $state(null); let editorOpen = $state(false); let preview = $state(null); let busy = $state(false), actionError = $state(''), refreshError = $state(''), notice = $state(''); let generation = 0; - const modeLabel = (mode: string) => ({ off: 'Off', 'dry-run': 'Dry run', 'create-only': 'Create only', 'create-destroy': 'Create and destroy' })[mode] ?? mode; + const modeLabel = (mode: string) => ({ off: 'Off', 'create-destroy': 'Create and destroy' })[mode] ?? mode; async function refresh(signal?: AbortSignal) { const version = ++generation; try { @@ -87,7 +87,7 @@
- + @@ -104,7 +104,7 @@

Preview · {modeLabel(preview.policy.mode)}

Target: {preview.decision.desired} running lobbies{preview.policy.mapSlug === '*' ? ' across all maps' : ''}. {preview.snapshot.keyPool.available} of {preview.snapshot.keyPool.total} game keys available. {preview.snapshot.members.length} existing or pending lobbies.

{preview.decision.action === 'wait' ? 'Wait' : `${preview.decision.action === 'create' ? 'Create' : 'Remove'} ${preview.decision.server}`}: {preview.decision.reason}

- {#if preview.policy.mode === 'create-only' || preview.policy.mode === 'create-destroy'}

Saving gives this policy exclusive control of the selected maps' lobby counts. Pause it before launching a competing collection or manual lifecycle action.

{/if} + {#if preview.policy.mode === 'create-destroy'}

Saving gives this policy exclusive control of the selected maps' lobby counts. Pause it before launching a competing collection or manual lifecycle action.

{/if} {#if preview.snapshot.members.length}
    {#each preview.snapshot.members as member}
  • {member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}
    • {/each}
{/if} {#each [...preview.blockers, ...preview.snapshot.blockers] as blocker}

{blocker}

{/each} @@ -112,22 +112,21 @@ {/if} {/if} - {#if !report.policies.length}

No autoscaling policies. Add a policy to observe a map or a whole game in dry run.

{/if} + {#if !report.policies.length}

No autoscaling policies. Add a policy to manage empty lobbies for a map or a whole game.

{/if} {#each report.policies as view (view.policy.id)}

{view.policy.mapSlug === '*' ? 'Every map' : zombieMapBySlug(view.policy.game, view.policy.mapSlug)?.friendly ?? view.policy.mapSlug} · {view.policy.game.toUpperCase()}

{modeLabel(view.policy.mode)}{#if canOperate}{/if}

{view.decision.occupied} occupied · {view.decision.empty} empty and ready · {view.decision.pending} pending · {view.decision.unknown} unknown or stopped

{#if view.unsettled.length}{/if}

{view.policy.mapSlug === '*' ? 'Per map: ' : ''}Minimum {view.policy.minimum} · Maximum {view.policy.maximum} · Spare target {view.policy.spares} · {view.snapshot.keyPool.available} / {view.snapshot.keyPool.total} game keys available

-

{view.policy.mode === 'dry-run' ? 'Hypothetical decision: ' : 'Current decision: '}{view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}

- {#if view.policy.mode === 'create-only' && view.decision.action === 'destroy'}

Removal is advisory in Create only mode.

{/if} +

Current decision: {view.decision.action}{view.decision.server ? ` ${view.decision.server}` : ''}. {view.decision.reason}

Last evaluated: {view.lastEvaluatedAt ? $timeFormat.dateTime(new Date(view.lastEvaluatedAt).toISOString()) : 'Not evaluated yet'}{view.policy.nextActionAt > Date.now() ? ` · Next action after ${$timeFormat.dateTime(new Date(view.policy.nextActionAt).toISOString())}` : ''}

Lobbies and evidence
    {#each view.snapshot.members as member}
  • {member.name} · {member.state}{member.blocker ? ` · ${member.blocker}` : ''}
    {member.source ?? 'No current source'} · {member.observedAt ? $timeFormat.dateTime(new Date(member.observedAt).toISOString()) : 'Unknown observation time'}{#if member.operationId} · Provisioning{/if}
    • {/each}
{#if !view.snapshot.members.length}

No registered or pending lobbies.

{/if}{#each view.snapshot.blockers as blocker}

{blocker}

{/each}
Recent decisions and measurements

{view.totals.observedMinutes.toFixed(1)} observed minutes · {view.totals.emptyLobbyHours.toFixed(2)} empty lobby-hours · {view.totals.deficitMinutes.toFixed(1)} minutes below spare target · {view.totals.unknownMinutes.toFixed(1)} unknown minutes

Last 31 days: {view.operations.creates} create requests · {view.operations.removals} removal requests · {view.operations.cancelled} cancelled · {view.operations.failed} failed

{#if view.operations.readinessSamples}

Request to verified readiness: mean {view.operations.readinessMeanSeconds?.toFixed(0)} seconds, maximum {view.operations.readinessMaxSeconds?.toFixed(0)} seconds across {view.operations.readinessSamples} completed creates. Includes queue time.

{/if} -

Up to 31 days of bounded observations; gaps are excluded. Dry run measures the actual fleet, not a simulated replacement lineup. Readiness does not certify browser visibility or successful joins.

+

Up to 31 days of bounded observations; gaps are excluded. Readiness does not certify browser visibility or successful joins.

    {#each view.history as item}
  1. {$timeFormat.dateTime(new Date(item.at).toISOString())} · {item.decision.action} {item.decision.server ?? ''}
    {item.decision.reason}{#if item.operationId} · {item.operationState ?? 'View operation'}{/if}
    2. {/each}
diff --git a/tests/unit/autoscaling-delete.test.ts b/tests/unit/autoscaling-delete.test.ts index 8ce0f4a..c5160b5 100644 --- a/tests/unit/autoscaling-delete.test.ts +++ b/tests/unit/autoscaling-delete.test.ts @@ -5,8 +5,8 @@ import { prepareNativeHost } from '../helpers/native-host.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); -const settings = { game: 't6', mapSlug: 'town', maximum: 1, mode: 'dry-run' }; -async function setup(mode = 'dry-run') { +const settings = { game: 't6', mapSlug: 'town', maximum: 1, mode: 'create-destroy' }; +async function setup(mode = 'create-destroy') { if (mode.startsWith('create')) await prepareNativeHost(); const { db } = await import('../../src/lib/server/db/index.js'); const scaling = await import('../../src/lib/server/autoscaling/index.js'); @@ -31,10 +31,10 @@ it('deletes policy evidence atomically while preserving lobbies, keys, and unrel expect(db().prepare('SELECT name FROM servers').all()).toEqual([{ name: 't6_zombie_town_1' }]); expect(db().prepare('SELECT COUNT(*) AS n FROM server_keys').get()).toEqual({ n: 2 }); expect(db().prepare('PRAGMA foreign_key_check').all()).toEqual([]); - expect(scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token).id).not.toBe(saved.id); + expect(scaling.saveScaling({ ...settings, mode: 'off' }, undefined, undefined, scaling.previewScaling({ ...settings, mode: 'off' }).token).id).not.toBe(saved.id); }); -it.each(['off', 'create-only', 'create-destroy'])('deletes a settled %s policy even after key loss and releases ownership', async mode => { +it.each(['off', 'create-destroy'])('deletes a settled %s policy even after key loss and releases ownership', async mode => { const { db, scaling, saved } = await setup(mode); db().prepare('DELETE FROM server_keys').run(); scaling.deleteScaling(saved.id, saved.revision); @@ -54,11 +54,11 @@ it('rejects stale revisions and preserves the latest settings', async () => { }); it('blocks deletion with pending intent, then keeps Provisioning history after pause and settlement', async () => { - const { db, scaling, saved } = await setup('create-only'); + const { db, scaling, saved } = await setup('create-destroy'); await scaling.runAutoscaling(); const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string }; expect(() => scaling.deleteScaling(saved.id, saved.revision)).toThrow('outstanding Provisioning'); - expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('create-only'); + expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('create-destroy'); const off = { ...settings, mode: 'off' }; const paused = scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token); scaling.deleteScaling(saved.id, paused.revision); diff --git a/tests/unit/autoscaling-dry-run.test.ts b/tests/unit/autoscaling-dry-run.test.ts deleted file mode 100644 index 65476e9..0000000 --- a/tests/unit/autoscaling-dry-run.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { afterEach, beforeEach, expect, it, vi } from 'vitest'; -import { newFixture, type Fixture } from '../helpers/fixture.js'; -import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js'; - -let fixture: Fixture; -beforeEach(async () => { - vi.resetModules(); fixture = await newFixture([]); - const { db } = await import('../../src/lib/server/db/index.js'); - for (const n of [3, 8, 9, 10]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `secret-${n}`); -}); -afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); }); -const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }; - -it('persists deduplicated dry-run decisions across restart without provisioning or reservations', async () => { - const { db, closeDb } = await import('../../src/lib/server/db/index.js'); - const scaling = await import('../../src/lib/server/autoscaling/index.js'); - const preview = scaling.previewScaling(settings); - const saved = scaling.saveScaling(settings, undefined, undefined, preview.token); - await scaling.runAutoscaling(); await scaling.runAutoscaling(); - closeDb(); - await scaling.runAutoscaling(); - const view = scaling.scalingReport().policies[0]; - expect(view.policy.id).toBe(saved.id); - expect(view.history).toHaveLength(1); - expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_1' }); - expect(JSON.stringify(view)).not.toContain('secret'); - for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 }); -}); - -it('rejects duplicate policies, stale edits and changed previews', async () => { - const scaling = await import('../../src/lib/server/autoscaling/index.js'); - const preview = scaling.previewScaling(settings); - const saved = scaling.saveScaling(settings, undefined, undefined, preview.token); - expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists'); - expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed'); - expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed'); - const { db } = await import('../../src/lib/server/db/index.js'); - const beforeKeyLoss = scaling.previewScaling(settings, saved.id, saved.revision); - db().prepare("DELETE FROM server_keys WHERE id='key-10'").run(); - expect(() => scaling.saveScaling(settings, saved.id, saved.revision, beforeKeyLoss.token)).toThrow('preview changed'); - expect(scaling.saveScaling(settings, saved.id, saved.revision, scaling.previewScaling(settings, saved.id, saved.revision).token).maximum).toBe(4); -}); - -it('pauses evaluations and prunes expired observations without deleting current history', async () => { - const scaling = await import('../../src/lib/server/autoscaling/index.js'); - scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); - await scaling.runAutoscaling(); - scaling.setScalingGlobalPause(true); await scaling.runAutoscaling(); - expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused'); - const { db } = await import('../../src/lib/server/db/index.js'); - const id = scaling.scalingReport().policies[0].history[0].id; - db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id); - scaling.pruneAutoscaling(); - expect(scaling.scalingReport().policies[0].history).toHaveLength(1); -}); - -it('uses fleet authorization for page, reads, and mutations', () => { - expect(routePermission('/autoscaling', 'GET')).toBe('viewer'); - expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer'); - expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false); - expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true); -}); diff --git a/tests/unit/autoscaling-game.test.ts b/tests/unit/autoscaling-game.test.ts index 03d57b4..9c3ea10 100644 --- a/tests/unit/autoscaling-game.test.ts +++ b/tests/unit/autoscaling-game.test.ts @@ -8,7 +8,7 @@ import { planScaling } from '../../src/lib/server/autoscaling/planner.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); -const settings = { game: 't6', mapSlug: '*', maximum: 2, mode: 'create-only' }; +const settings = { game: 't6', mapSlug: '*', maximum: 2, mode: 'create-destroy' }; const maps = zombieMaps('t6'); it('satisfies a T5 game-wide policy with the ten standard Zombies maps', () => { diff --git a/tests/unit/autoscaling-keys.test.ts b/tests/unit/autoscaling-keys.test.ts index f988048..eef0672 100644 --- a/tests/unit/autoscaling-keys.test.ts +++ b/tests/unit/autoscaling-keys.test.ts @@ -5,16 +5,16 @@ import { prepareNativeHost } from '../helpers/native-host.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); -const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'dry-run' }; +const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' }; async function setup() { + await prepareNativeHost(); const { db } = await import('../../src/lib/server/db/index.js'); const scaling = await import('../../src/lib/server/autoscaling/index.js'); const { saveServerKey, reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); return { db, scaling, saveServerKey, reserveServerKey }; } -it.each(['off', 'dry-run', 'create-only', 'create-destroy'])('saves a %s policy without preallocating keys for its maximum', async mode => { - if (mode.startsWith('create')) await prepareNativeHost(); +it.each(['off', 'create-destroy'])('saves a %s policy without preallocating keys for its maximum', async mode => { const { db, scaling } = await setup(); const input = { ...settings, mode, maximum: 20 }; const preview = scaling.previewScaling(input); @@ -65,12 +65,11 @@ it('rechecks game pool deletion and reservations after preview', async () => { }); it('shares capacity between live map policies and admits the job holding the last key', async () => { - await prepareNativeHost(); const { db, scaling, saveServerKey } = await setup(); saveServerKey('t6', 'game-key'); saveServerKey('t5', 'other-game-key'); for (const mapSlug of ['town', 'farm']) { - const input = { ...settings, mapSlug, mode: 'create-only' }; + const input = { ...settings, mapSlug, mode: 'create-destroy' }; scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token); } expect(await scaling.runAutoscaling()).toBe(true); diff --git a/tests/unit/autoscaling-live.test.ts b/tests/unit/autoscaling-live.test.ts index 90a8af8..5f226b4 100644 --- a/tests/unit/autoscaling-live.test.ts +++ b/tests/unit/autoscaling-live.test.ts @@ -5,7 +5,7 @@ import { prepareNativeHost } from '../helpers/native-host.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); -const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-only' }; +const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' }; async function setup() { await prepareNativeHost(); const { db } = await import('../../src/lib/server/db/index.js'); @@ -77,8 +77,11 @@ it('operator cancellation pauses the policy and does not immediately create a re it('exact matching cannot remove an owned map absent from the collection', async () => { const town = 't6_zombie_town_1'; - await prepareNativeHost([town]); + const { host } = await prepareNativeHost([town]); const { db } = await import('../../src/lib/server/db/index.js'); + const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js'); + db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?") + .run(JSON.stringify([autoscalingRemovalCapability]), host.id); const at = new Date().toISOString(); db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at); for (const n of [1, 2]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `key-${n}`); diff --git a/tests/unit/autoscaling-planner.test.ts b/tests/unit/autoscaling-planner.test.ts index 16712c2..23b4d98 100644 --- a/tests/unit/autoscaling-planner.test.ts +++ b/tests/unit/autoscaling-planner.test.ts @@ -4,7 +4,7 @@ import { emptyObservation, planScaling } from '../../src/lib/server/autoscaling/ import { mapIdentity } from '../../src/lib/server/autoscaling/snapshot.js'; const now = 2_000_000; -const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' }; +const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' }; const name = (n: number) => `t6_zombie_town_${n}`; const member = (n: number, state: ScalingMember['state'] = 'empty'): ScalingMember => ({ name: name(n), identity: `instance:${n}`, state, players: state === 'occupied' ? 1 : state === 'empty' ? 0 : null, observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null }); const snapshot = (...members: ScalingMember[]): ScalingSnapshot => ({ members, reservedNames: members.map(m => m.name), keyPool: { total: 4, available: 4 - members.length }, blockers: [] }); diff --git a/tests/unit/autoscaling-policies.test.ts b/tests/unit/autoscaling-policies.test.ts new file mode 100644 index 0000000..7459504 --- /dev/null +++ b/tests/unit/autoscaling-policies.test.ts @@ -0,0 +1,116 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { prepareNativeHost } from '../helpers/native-host.js'; +import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js'; + +let fixture: Fixture; +beforeEach(async () => { + vi.resetModules(); fixture = await newFixture([]); + await prepareNativeHost(); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const n of [3, 8, 9, 10]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `secret-${n}`); +}); +afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); }); +const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' }; + +it('persists deduplicated paused decisions across restart without provisioning or reservations', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const preview = scaling.previewScaling(settings); + const saved = scaling.saveScaling(settings, undefined, undefined, preview.token); + scaling.setScalingGlobalPause(true); + await scaling.runAutoscaling(); await scaling.runAutoscaling(); + closeDb(); + await scaling.runAutoscaling(); + const view = scaling.scalingReport().policies[0]; + expect(view.policy.id).toBe(saved.id); + expect(view.history).toHaveLength(1); + expect(view.decision).toMatchObject({ action: 'wait', server: null }); + expect(JSON.stringify(view)).not.toContain('secret'); + for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 }); +}); + +it('rejects duplicate policies, stale edits and changed previews', async () => { + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const preview = scaling.previewScaling(settings); + const saved = scaling.saveScaling(settings, undefined, undefined, preview.token); + expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('overlapping policy'); + expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed'); + expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed'); + const { db } = await import('../../src/lib/server/db/index.js'); + const beforeKeyLoss = scaling.previewScaling(settings, saved.id, saved.revision); + db().prepare("DELETE FROM server_keys WHERE id='key-10'").run(); + expect(() => scaling.saveScaling(settings, saved.id, saved.revision, beforeKeyLoss.token)).toThrow('preview changed'); + expect(scaling.saveScaling(settings, saved.id, saved.revision, scaling.previewScaling(settings, saved.id, saved.revision).token).maximum).toBe(4); +}); + +it('pauses evaluations and prunes expired observations without deleting current history', async () => { + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('DELETE FROM server_keys').run(); + scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + await scaling.runAutoscaling(); + scaling.setScalingGlobalPause(true); await scaling.runAutoscaling(); + expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused'); + const id = scaling.scalingReport().policies[0].history[0].id; + db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id); + scaling.pruneAutoscaling(); + expect(scaling.scalingReport().policies[0].history).toHaveLength(1); +}); + +it('uses fleet authorization for page, reads, and mutations', () => { + expect(routePermission('/autoscaling', 'GET')).toBe('viewer'); + expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer'); + expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false); + expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true); +}); + +it.each(['dry-run', 'create-only'])('rejects removed %s mode through the API', async mode => { + const { POST } = await import('../../src/routes/api/autoscaling/+server.js'); + for (const action of ['preview', 'save']) { + const response = await POST({ request: new Request('http://localhost/api/autoscaling', { + method: 'POST', body: JSON.stringify({ action, settings: { ...settings, mode } }) + }) }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: 'Choose a valid autoscaling mode.' }); + } +}); + +it.each(['dry-run', 'create-only'])('turns saved %s policies off once on upgrade', async mode => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + const otherSettings = { ...settings, mapSlug: 'farm' }; + const other = scaling.saveScaling(otherSettings, undefined, undefined, scaling.previewScaling(otherSettings).token); + db().prepare('UPDATE autoscaling_policies SET mode=?,settings=json_set(settings,\'$.mode\',?) WHERE id=?').run(mode, mode, saved.id); + closeDb(); + const migrated = scaling.getScalingPolicy(saved.id)!; + expect(migrated).toMatchObject({ mode: 'off', revision: saved.revision + 1, maximum: saved.maximum, + pausedReason: expect.stringContaining('mode was removed') }); + expect(db().prepare('SELECT owns_map,json_extract(settings,\'$.mode\') AS mode FROM autoscaling_policies WHERE id=?').get(saved.id)) + .toEqual({ owns_map: 0, mode: 'off' }); + expect(scaling.getScalingPolicy(other.id)).toEqual(other); + closeDb(); + expect(scaling.getScalingPolicy(saved.id)).toEqual(migrated); + const resumed = scaling.saveScaling(settings, migrated.id, migrated.revision, + scaling.previewScaling(settings, migrated.id, migrated.revision).token); + expect(resumed).toMatchObject({ mode: 'create-destroy', pausedReason: null }); +}); + +it.each([false, true])('reconciles removed-mode creation while preserving submitted work: %s', async submitted => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + await scaling.runAutoscaling(); + const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string }; + db().prepare("UPDATE autoscaling_policies SET mode='create-only',settings=json_set(settings,'$.mode','create-only') WHERE id=?").run(saved.id); + if (submitted) db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), id); + closeDb(); + expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('off'); + expect(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(saved.id)).toEqual({ owns_map: 1 }); + expect(await scaling.runAutoscaling()).toBe(false); + expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: submitted ? 'queued' : 'cancelled' }); + expect(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(saved.id)).toEqual({ owns_map: submitted ? 1 : 0 }); + expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: submitted ? 1 : 0 }); + expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()).toEqual({ n: 1 }); +}); diff --git a/tests/unit/autoscaling-snapshot.test.ts b/tests/unit/autoscaling-snapshot.test.ts index 26dd482..06351e3 100644 --- a/tests/unit/autoscaling-snapshot.test.ts +++ b/tests/unit/autoscaling-snapshot.test.ts @@ -7,7 +7,7 @@ let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); const town = 't6_zombie_town_1', farm = 't6_zombie_farm_1'; -const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' }; +const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' }; it('separates variants, deduplicates pending identities, and initializes additive schema', async () => { await prepareNativeHost([town, farm]);