Complete provisioning job detail pages and lifecycle evidence
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_1';
|
||||
const status = (map = 'zombie_theater') => ({ ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map, gametype: 'zom', roster: [] } });
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
||||
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
}
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(status());
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function register() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
}
|
||||
async function detail(id: string) {
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
return getDeployment(id)!;
|
||||
}
|
||||
async function telemetry(players: number, map = 'zombie_theater') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const receivedAt = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
||||
ON CONFLICT(server) DO UPDATE SET players=excluded.players,received_at=excluded.received_at,payload=excluded.payload`)
|
||||
.run(server, players, receivedAt, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt }));
|
||||
return receivedAt;
|
||||
}
|
||||
|
||||
describe('provisioning decision evidence', () => {
|
||||
it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
|
||||
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { script } = createScript({ name: 'New promotion', targetRelPath: 'promoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('promoted while queued') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress?.recipeStatus).toBe('attempted');
|
||||
expect((await detail(job.deploymentId)).recipe?.files.some(file => file.path === 'promoted.gsc')).toBe(true);
|
||||
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce(status('zombie_moon'));
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'verify-map', blocker: { code: 'map-mismatch' },
|
||||
checks: [{ key: 'container', result: 'pass' }, { key: 'game', result: 'pass' }, { key: 'map', result: 'waiting', expected: 'zombie_theater', actual: 'zombie_moon' }] });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'container recovering' });
|
||||
await processCreations();
|
||||
expect((await detail(job.deploymentId)).progress?.checks).toEqual([expect.objectContaining({ key: 'container', result: 'waiting' })]);
|
||||
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
const receivedAt = await telemetry(0);
|
||||
await processCreations();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment.state).toBe('done');
|
||||
expect(completed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'game', source: 'telemetry', result: 'pass', receivedAt }));
|
||||
expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'verify-map' }));
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps an uncertain failed request immutable while observing and releasing its exact recovery slot', async () => {
|
||||
await register();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'response lost' });
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
await processProvisioning();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'held', operationId: job.deploymentId } });
|
||||
const failureEvents = failed.events.filter(event => event.kind === 'outcome');
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await telemetry(0);
|
||||
await processProvisioning();
|
||||
const recovered = await detail(job.deploymentId);
|
||||
expect(recovered.deployment).toMatchObject({ state: 'failed', finished_at: failed.deployment.finished_at });
|
||||
expect(recovered.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'released', operationId: job.deploymentId } });
|
||||
expect(recovered.events.filter(event => event.kind === 'outcome')).toEqual(failureEvents);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('records occupied and unknown evidence without treating either as zero or resetting eligibility timing', async () => {
|
||||
await register();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null);
|
||||
await processProvisioning();
|
||||
const unknown = await detail(job.deploymentId);
|
||||
expect(unknown.progress?.checks).toEqual([expect.objectContaining({ key: 'occupancy', actual: null, source: null, result: 'unknown' })]);
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await telemetry(3);
|
||||
await processProvisioning();
|
||||
expect((await detail(job.deploymentId)).progress).toMatchObject({ phaseStartedAt: unknown.progress?.phaseStartedAt,
|
||||
blocker: { code: 'players-present' }, checks: [{ key: 'occupancy', actual: 3, source: 'telemetry' }] });
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await telemetry(2);
|
||||
await processProvisioning();
|
||||
expect((await detail(job.deploymentId)).progress?.phaseStartedAt).toBe(unknown.progress?.phaseStartedAt);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves failed cleanup evidence through same-ID resume without another destructive command', async () => {
|
||||
await register();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { resumeCleanup } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'cleanup interrupted'); END");
|
||||
const job = destroyServer(server);
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
await processDestructions();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment.state).toBe('failed');
|
||||
expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
|
||||
expect(failed.progress?.checkpoints.some(point => point.key === 'cleanup')).toBe(false);
|
||||
const failure = failed.events.find(event => event.kind === 'outcome')!;
|
||||
db().exec('DROP TRIGGER fail_cleanup');
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
expect(resumeCleanup(job.deploymentId)).toBe(true);
|
||||
await processDestructions();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment).toMatchObject({ id: job.deploymentId, state: 'done', server_exists: 0 });
|
||||
expect(completed.events).toContainEqual(failure);
|
||||
expect(completed.events.some(event => event.kind === 'resumed')).toBe(true);
|
||||
expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'cleanup' }));
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('links the actual removal dependency while later eligible creation bypasses it', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' };
|
||||
const first = queueCreateServer(input), second = queueCreateServer(input);
|
||||
if (!first.ok || !second.ok) throw new Error('Could not queue creation');
|
||||
const removal = beginDeployment('old-instance', 'destroy');
|
||||
db().prepare('INSERT INTO server_creation_dependencies VALUES (?, ?)').run(first.deploymentId, removal);
|
||||
await processCreations();
|
||||
expect((await detail(first.deploymentId)).progress?.blocker).toMatchObject({ code: 'removal-dependency', href: `/provisioning/${removal}` });
|
||||
expect((await detail(first.deploymentId)).deployment.state).toBe('queued');
|
||||
expect((await detail(second.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
});
|
||||
|
||||
it.each(['create', 'delete'] as const)('records uncertainty when the %s reply throws without inventing a successful checkpoint', async command => {
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { queueCreateServer, processCreations, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
let id: string;
|
||||
if (command === 'create') {
|
||||
vi.mocked(komodo.komodoCreateDeployment).mockRejectedValueOnce(new Error('reply lost'));
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
id = job.deploymentId;
|
||||
await processCreations();
|
||||
expect(komodo.komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
} else {
|
||||
await register();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodo.komodoDeleteDeployment).mockRejectedValueOnce(new Error('reply lost'));
|
||||
const job = destroyServer(server);
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
id = job.deploymentId;
|
||||
await processDestructions();
|
||||
expect(komodo.komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
const failed = await detail(id);
|
||||
expect(failed.deployment.state).toBe('failed');
|
||||
expect(failed.progress?.command).toMatchObject({ state: 'uncertain', message: expect.stringContaining('reply lost') });
|
||||
expect(failed.progress?.checkpoints.some(point => point.key === (command === 'create' ? 'create-deployment' : 'delete-deployment'))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['destroy', 'rebuild'] as const)('retains the resumable %s checkpoint when a beta arrives after confirmed remote deletion', async action => {
|
||||
await register();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { resumeCleanup, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodoDeleteDeployment).mockImplementationOnce(async () => {
|
||||
const { script, version } = createScript({ name: 'Late beta', targetRelPath: 'beta.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta') });
|
||||
const beta = db().prepare("INSERT INTO betas (script_id,script_version_id,trigger_kind,state) VALUES (?,?,'manual','running')").run(script.id, version.id).lastInsertRowid;
|
||||
db().prepare("INSERT INTO beta_servers (beta_id,server,had_original,stage) VALUES (?,?,0,'ready')").run(beta, server);
|
||||
return { ok: true };
|
||||
});
|
||||
const job = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
|
||||
if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
|
||||
await (action === 'destroy' ? processDestructions() : processProvisioning());
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER });
|
||||
expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
|
||||
expect(action === 'destroy' ? failed.canResumeCleanup : failed.canResumeRebuild).toBe(true);
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back'").run();
|
||||
expect(action === 'destroy' ? resumeCleanup(job.deploymentId) : resumeRebuild(job.deploymentId)).toBe(true);
|
||||
await (action === 'destroy' ? processDestructions() : processProvisioning());
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe(action === 'destroy' ? 'done' : 'queued');
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('ignores occupancy evidence that returns after the request was cancelled', async () => {
|
||||
await register();
|
||||
const { requestLifecycle, cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const job = requestLifecycle(server, 'restart');
|
||||
if (!job.ok) throw new Error(job.error);
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
expect(cancelProvisioning(job.deploymentId)).toBe(true);
|
||||
return 4;
|
||||
});
|
||||
await processProvisioning();
|
||||
const cancelled = await detail(job.deploymentId);
|
||||
expect(cancelled.deployment.state).toBe('cancelled');
|
||||
expect(cancelled.progress).toMatchObject({ checks: [], blocker: null });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not attach replacement health to a deleted creation after a delayed probe', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await register();
|
||||
return { ok: true, state: 'running' };
|
||||
});
|
||||
await processCreations();
|
||||
const old = await detail(job.deploymentId);
|
||||
expect(old.deployment.server_deleted_at).toBeTruthy();
|
||||
expect(old.progress?.checks).toEqual([]);
|
||||
expect(old.progress?.checkpoints.some(point => point.key.startsWith('verify-'))).toBe(false);
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user