Allow manual fleet actions while autoscaling is paused
This commit is contained in:
@@ -33,7 +33,7 @@ Existing policies saved in the removed Dry run or Create only modes switch to Of
|
||||
|
||||
## Ownership and blockers
|
||||
|
||||
A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. Competing collection launches and manual Create, Start, Stop, or Destroy requests are rejected with a policy link. Restart and Rebuild remain available through the usual provisioning checks. Their pending work and unresolved recovery hold scaling decisions for the affected map, so temporary downtime does not trigger replacement lobbies. Exact collection matching also checks managed maps absent from the collection. To hand control back, set the policy to Off and wait for unsettled operations to finish.
|
||||
A live policy owns the lobby counts for its selected maps. Overlapping live game-wide and map-specific policies are rejected. Game-wide rules include every catalog map and apply each map’s targets independently while sharing the game’s key pool. Pool capacity is a game-wide limit; targets across maps are not guaranteed when all keys are in use. While autoscaling is active, competing collection launches and manual Create, Start, Stop, or Destroy requests are rejected with a policy link. Pausing globally or pausing the policy allows these manual actions through their usual provisioning checks. Restart and Rebuild remain available through the usual provisioning checks. Their pending work and unresolved recovery hold scaling decisions for the affected map, so temporary downtime does not trigger replacement lobbies. Exact collection matching also checks managed maps absent from the collection.
|
||||
|
||||
Unknown or stale occupancy, stopped or unhealthy members, active betas, campaigns, maintenance, host drains, and unresolved recovery hold the policy's decisions. Autoscaling does not repair these conditions itself. The page and linked Provisioning operation explain the current blocker, including capacity and game key waits.
|
||||
|
||||
@@ -46,7 +46,7 @@ Current positive occupancy from either telemetry or RCON vetoes an empty claim.
|
||||
|
||||
## Pause and cancellation
|
||||
|
||||
Use **Pause all** to stop new decisions globally, or **Pause** on one policy. Use **Resume** to enable it again; the usual activation checks still apply. A global pause retains each live policy's ownership; setting it Off releases ownership once its outstanding operation settles. Resuming the global switch preserves each policy's On or Off setting.
|
||||
Use **Pause all** to stop new decisions globally, or **Pause** on one policy. Use **Resume** to enable it again; the usual activation checks still apply. A global pause retains each live policy's ownership to prevent overlapping policies, but allows manual fleet, server, and collection actions. Policies paused after a failure also allow manual actions. Work already in progress still uses the normal per-server provisioning checks. Setting a policy Off releases ownership once its outstanding operation settles. Resuming the global switch preserves each policy's On or Off setting and restores the manual-action guard for active policies.
|
||||
|
||||
Unsubmitted creates and queued removals are withdrawn where possible. Cancelling an autoscaling operation from Provisioning also turns its policy Off so it cannot immediately request a replacement.
|
||||
|
||||
|
||||
@@ -1,20 +1,37 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { liveScaling, type ScalingPolicy } from '../../autoscaling.js';
|
||||
import { mapIdentity } from './snapshot.js';
|
||||
|
||||
function owningPolicy(game: string, mapSlug: string) {
|
||||
return db().prepare("SELECT id,mode,paused_reason FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1")
|
||||
.get(game, mapSlug, mapSlug) as { id: string; mode: ScalingPolicy['mode']; paused_reason: string | null } | undefined;
|
||||
}
|
||||
/** Durable ownership prevents overlapping policies even while scaling is paused. */
|
||||
export function scalingOwner(game: string, mapSlug: string): string | null {
|
||||
return (db().prepare("SELECT id FROM autoscaling_policies WHERE game=? AND (map_slug=? OR map_slug='*' OR ?='*') AND owns_map=1").get(game, mapSlug, mapSlug) as { id: string } | undefined)?.id ?? null;
|
||||
return owningPolicy(game, mapSlug)?.id ?? null;
|
||||
}
|
||||
const ownershipMessage = (owner: string) => `Autoscaling manages this map. Pause its policy before changing the lineup: /autoscaling#${owner}`;
|
||||
function manualLineupBlocker(game: string, mapSlug: string): string | null {
|
||||
const owner = owningPolicy(game, mapSlug);
|
||||
if (!owner || !liveScaling(owner.mode) || owner.paused_reason) return null;
|
||||
if ((db().prepare("SELECT value FROM settings WHERE key='autoscaling_paused'").get() as { value: string } | undefined)?.value === '1') return null;
|
||||
return ownershipMessage(owner.id);
|
||||
}
|
||||
export function scalingOwnershipBlocker(server: string): string | null {
|
||||
const identity = mapIdentity(getServer(server) ?? { name: server });
|
||||
const owner = identity && scalingOwner(identity.game, identity.mapSlug);
|
||||
return owner ? `Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}` : null;
|
||||
return identity ? manualLineupBlocker(identity.game, identity.mapSlug) : null;
|
||||
}
|
||||
export function assertScalingOwnership(game: string, mapSlug: string, decisionId?: string): void {
|
||||
if (!decisionId) {
|
||||
const blocker = manualLineupBlocker(game, mapSlug);
|
||||
if (blocker) throw new Error(blocker);
|
||||
return;
|
||||
}
|
||||
const owner = scalingOwner(game, mapSlug);
|
||||
if (!owner) { if (decisionId) throw new Error('Autoscaling no longer owns this map.'); return; }
|
||||
if (decisionId && db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
|
||||
throw new Error(`Autoscaling manages this map. Pause its policy and settle pending work before changing the lineup: /autoscaling#${owner}`);
|
||||
if (!owner) throw new Error('Autoscaling no longer owns this map.');
|
||||
if (db().prepare('SELECT 1 FROM autoscaling_decisions WHERE id=? AND policy_id=? AND resolved=0 AND operation_id IS NULL').get(decisionId, owner)) return;
|
||||
throw new Error(ownershipMessage(owner));
|
||||
}
|
||||
/** Operator cancellation is a policy pause, not a fresh scaling signal. */
|
||||
export function pauseScalingForOperation(operationId: string): void {
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { prepareNativeHost } from '../helpers/native-host.js';
|
||||
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { await fixture.cleanup(); });
|
||||
const names = ['t6_zombie_town_1', 't6_zombie_town_2'];
|
||||
const settings = { game: 't6', mapSlug: 'town', maximum: 3, mode: 'create-destroy' };
|
||||
|
||||
async function setup(scope = 'town') {
|
||||
const { host } = await prepareNativeHost(names);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
||||
const ownership = await import('../../src/lib/server/autoscaling/ownership.js');
|
||||
const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js');
|
||||
db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?")
|
||||
.run(JSON.stringify([autoscalingRemovalCapability]), host.id);
|
||||
const at = new Date().toISOString();
|
||||
for (const name of names) {
|
||||
db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
|
||||
VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(name, at, at, at);
|
||||
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(name, `key-${name}`);
|
||||
db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name);
|
||||
}
|
||||
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')").run();
|
||||
const policy = scaling.saveScaling({ ...settings, mapSlug: scope });
|
||||
const { POST } = await import('../../src/routes/api/provisioning/+server.js');
|
||||
const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
|
||||
const { queueFleetAction } = await import('../../src/lib/ui/fleet-actions.js');
|
||||
const request: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input), 'http://localhost');
|
||||
if (init?.method === 'DELETE') return DELETE({ params: { name: decodeURIComponent(url.pathname.split('/').at(-1)!) } });
|
||||
return POST({ request: new Request(url, init) });
|
||||
};
|
||||
const fleet = (action: 'start' | 'stop' | 'destroy', servers = names) => queueFleetAction(action, servers, request);
|
||||
return { db, scaling, ownership, policy, fleet };
|
||||
}
|
||||
|
||||
it.each(['town', '*'])('blocks fleet lineup changes while the %s policy is active', async scope => {
|
||||
const { db, fleet } = await setup(scope);
|
||||
for (const action of ['start', 'stop', 'destroy'] as const) {
|
||||
expect(await fleet(action)).toEqual({ queued: [], failed: names.map(server => ({ server, error: expect.stringContaining('Autoscaling') })) });
|
||||
}
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['start', 'stop', 'destroy'] as const)('allows fleet %s during global pause and restores the guard on resume', async action => {
|
||||
const { db, scaling, ownership, policy, fleet } = await setup();
|
||||
scaling.setScalingGlobalPause(true);
|
||||
expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
|
||||
expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
|
||||
expect(scaling.evaluateScaling({ ...settings, mapSlug: '*' }).blockers.join(' ')).toContain('overlapping');
|
||||
expect(() => scaling.saveScaling({ ...settings, mapSlug: '*' })).toThrow('overlapping');
|
||||
scaling.setScalingGlobalPause(false);
|
||||
expect((await fleet(action)).failed).toHaveLength(2);
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
scaling.setScalingGlobalPause(true);
|
||||
const result = await fleet(action);
|
||||
expect(result.failed).toEqual([]);
|
||||
expect(result.queued.map(job => job.server)).toEqual(names);
|
||||
expect(db().prepare('SELECT server,action,state FROM server_deployments ORDER BY server').all())
|
||||
.toEqual(names.map(server => ({ server, action, state: 'waiting' })));
|
||||
});
|
||||
|
||||
it('keeps internal decision authorization scoped to its owning policy during global pause', async () => {
|
||||
const { db, scaling, ownership } = await setup();
|
||||
const other = scaling.saveScaling({ ...settings, mapSlug: 'farm' });
|
||||
scaling.setScalingGlobalPause(true);
|
||||
db().prepare(`INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence,resolved)
|
||||
VALUES('other-decision',?,?,?,?,'fixture','{}','{}',0)`).run(other.id, other.revision, Date.now(), Date.now());
|
||||
expect(() => ownership.assertScalingOwnership('t6', 'town', 'forged-decision')).toThrow('Autoscaling');
|
||||
expect(() => ownership.assertScalingOwnership('t6', 'town', 'other-decision')).toThrow('Autoscaling');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
expect(() => destroyServer(names[0], 'autoscaling:test', 'other-decision')).toThrow('Autoscaling');
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['off', 'paused reason'])('permits manual requests while an individually paused policy retains ownership: %s', async pause => {
|
||||
const { db, ownership, policy, fleet } = await setup();
|
||||
// Ownership may remain while a paused policy still has work to reconcile.
|
||||
if (pause === 'off') db().prepare("UPDATE autoscaling_policies SET mode='off' WHERE id=?").run(policy.id);
|
||||
else db().prepare("UPDATE autoscaling_policies SET paused_reason='Inspect the failed operation.' WHERE id=?").run(policy.id);
|
||||
expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
|
||||
expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
|
||||
expect(() => ownership.assertScalingOwnership('t6', 'town', 'invalid-decision')).toThrow('Autoscaling');
|
||||
expect((await fleet('stop', [names[0]])).failed).toEqual([]);
|
||||
expect((await fleet('destroy', [names[1]])).failed).toEqual([]);
|
||||
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' }))
|
||||
.toMatchObject({ ok: true, name: 't6_zombie_town_3' });
|
||||
});
|
||||
|
||||
it('rechecks autoscaling when submitting a manual creation preview made during global pause', async () => {
|
||||
const { db, scaling } = await setup();
|
||||
const { previewCreation, queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const input = { game: 't6' as const, gameMode: 'zombie' as const, mapSlug: 'town' };
|
||||
expect(queueCreateServer(input)).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
|
||||
scaling.setScalingGlobalPause(true);
|
||||
const preview = previewCreation(input);
|
||||
expect(preview).toMatchObject({ ready: true, name: 't6_zombie_town_3' });
|
||||
scaling.setScalingGlobalPause(false);
|
||||
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
scaling.setScalingGlobalPause(true);
|
||||
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: true, name: preview.name });
|
||||
});
|
||||
|
||||
it.each(['fill', 'exact'] as const)('allows collection %s during global pause and rejects a paused preview after resume', async mode => {
|
||||
const { db, scaling } = await setup();
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const blueprint = saveBlueprint({ name: 'Town lineup', entries: [{ game: 't6', mapSlug: 'town', count: mode === 'fill' ? 3 : 1 }] });
|
||||
expect(previewCollection(blueprint.id, mode).blockers.join(' ')).toContain('Autoscaling');
|
||||
scaling.setScalingGlobalPause(true);
|
||||
const preview = previewCollection(blueprint.id, mode);
|
||||
expect(preview).toMatchObject({ ready: true, blockers: [] });
|
||||
if (mode === 'fill') expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_3']);
|
||||
else expect(preview.removals.map(server => server.name)).toEqual([names[1]]);
|
||||
scaling.setScalingGlobalPause(false);
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
scaling.setScalingGlobalPause(true);
|
||||
expect(queueCollection({ previewId: preview.id })).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
||||
expect(db().prepare('SELECT server,action FROM server_deployments').all()).toEqual([
|
||||
{ server: mode === 'fill' ? 't6_zombie_town_3' : names[1], action: mode === 'fill' ? 'create' : 'destroy' }
|
||||
]);
|
||||
});
|
||||
|
||||
it('preserves provisioning conflicts while letting other paused fleet members proceed', async () => {
|
||||
const { db, scaling, fleet } = await setup();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const pending = requestLifecycle(names[0], 'restart');
|
||||
expect(pending.ok).toBe(true);
|
||||
scaling.setScalingGlobalPause(true);
|
||||
expect(await fleet('stop', [names[0]])).toEqual({ queued: [], failed: [
|
||||
{ server: names[0], error: expect.stringContaining('Another provisioning operation') }
|
||||
] });
|
||||
const result = await fleet('destroy');
|
||||
expect(result.failed).toEqual([{ server: names[0], error: expect.stringContaining('current provisioning operation') }]);
|
||||
expect(result.queued).toEqual([{ server: names[1], deploymentId: expect.any(String) }]);
|
||||
expect(db().prepare('SELECT server,action FROM server_deployments ORDER BY server').all()).toEqual([
|
||||
{ server: names[0], action: 'restart' }, { server: names[1], action: 'destroy' }
|
||||
]);
|
||||
});
|
||||
Reference in New Issue
Block a user