feat: add provisioning lifecycle actions
Rename Deployments to Provisioning with compatible redirects and APIs. Add durable Start, Restart, Stop, and Rebuild operations alongside Create and Destroy; rebuild wipes the instance and recreates it in the same slot and port. Add lifecycle, migration, recovery, and Komodo command tests. Type checks and production build pass; the full suite retains a telemetry assertion failure reproduced on the unchanged branch. Refs #39
This commit is contained in:
@@ -0,0 +1,246 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import { readFile, writeFile, access } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_2';
|
||||
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
// Exercise the upgrade path, with history and references present before migration.
|
||||
const old = new Database(fixture.dbPath);
|
||||
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'"));
|
||||
old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
|
||||
old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
|
||||
old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
|
||||
old.close();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
vi.mocked(kick).mockResolvedValue(undefined);
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function operation(id: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
async function queue(action: 'start' | 'restart' | 'stop' | 'rebuild') {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const result = requestLifecycle(server, action);
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
return result.deploymentId;
|
||||
}
|
||||
|
||||
describe('provisioning operations', () => {
|
||||
it('migrates the action constraint without losing row ordering, activity, or foreign keys', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await queue('start');
|
||||
closeDb();
|
||||
expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
|
||||
expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
|
||||
expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
});
|
||||
|
||||
it('starts without an RCON empty check and verifies recovery after the grace period', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('start');
|
||||
await processLifecycle();
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledWith(server, 'start');
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ action: 'start', state: 'awaiting-health' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deduplicates requests, rejects conflicting actions, and waits for a fresh empty reading', async () => {
|
||||
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('restart');
|
||||
expect(requestLifecycle(server, 'restart')).toEqual({ ok: true, deploymentId: id });
|
||||
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
|
||||
expect(destroyServer(server)).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
for (const players of [4, null]) {
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting' });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
}
|
||||
await processLifecycle();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not repeat an uncertain restart and retains its recovery slot', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
const id = await queue('restart');
|
||||
await processLifecycle(); await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed', error: 'komodo: timeout' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
});
|
||||
|
||||
it('keeps stop pending on unknown health and completes only on a confirmed stopped state', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { komodoInspectHealthy, komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const id = await queue('stop');
|
||||
await processLifecycle();
|
||||
closeDb();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'network error' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
||||
await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rebuilds from nothing in the same slot and port, then verifies the fresh instance', async () => {
|
||||
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { processCreations, nextLobbyNumber, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const sentinel = join(fixture.t5, server, 'old-instance.dat');
|
||||
await writeFile(sentinel, 'must not survive');
|
||||
const id = await queue('rebuild');
|
||||
// Promotions after queueing must be included in the fresh instance.
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { script } = createScript({ name: 'Fresh promoted script', targetRelPath: 'fresh.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('fresh content') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processLifecycle();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledWith(server);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledWith(server);
|
||||
await expect(access(sentinel)).rejects.toThrow();
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
expect(await operation(id)).toMatchObject({ state: 'queued', action: 'rebuild' });
|
||||
expect(nextAvailablePort()).toBe(28961);
|
||||
expect(nextLobbyNumber('t5_zombie_theater', ['t5_zombie_theater_1'])).toBe(3);
|
||||
expect(requestLifecycle(server, 'rebuild')).toEqual({ ok: true, deploymentId: id });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledWith(expect.objectContaining({ name: server, ports: ['28960:28960/udp', '28960:28960/tcp'] }));
|
||||
await expect(access(sentinel)).rejects.toThrow();
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
|
||||
it('rejects a rebuild without a saved key before touching the existing instance', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
removeServerKey(server);
|
||||
expect(requestLifecycle(server, 'rebuild')).toMatchObject({ ok: false, error: expect.stringContaining('Save a server key') });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains the rebuild cleanup checkpoint and reservation while waiting for capacity', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { beginDeployment, deploymentStage, finishDeployment, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('rebuild');
|
||||
// Model a crash after confirmed remote deletion and registry cleanup.
|
||||
deploymentStage(id, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
finishDeployment(id, { ok: false, message: 'cleanup interrupted' });
|
||||
expect(nextAvailablePort()).toBe(28961);
|
||||
expect(resumeRebuild(id)).toBe(true);
|
||||
const occupied = beginDeployment('another-server', 'create');
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
|
||||
closeDb();
|
||||
finishDeployment(occupied, { ok: true });
|
||||
await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'queued' });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows explicit destruction of a stalled startup and records the interrupted verification', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('start');
|
||||
await processLifecycle();
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processDestructions();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed', error: 'Server destroyed before operation verification completed' });
|
||||
});
|
||||
|
||||
it('validates lifecycle API input and returns the durable operation ID', async () => {
|
||||
const { POST, GET } = await import('../../src/routes/api/provisioning/+server.js');
|
||||
const submit = (body: unknown) => POST({ request: new Request('http://localhost/api/provisioning', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
|
||||
}) });
|
||||
for (const body of [null, { server, action: 'erase' }, { server: 5, action: 'start' }]) expect((await submit(body)).status).toBe(400);
|
||||
const result = await submit({ server, action: 'start' });
|
||||
expect(result.status).toBe(202);
|
||||
const { deploymentId } = await result.json();
|
||||
expect((await GET().json()).deployments).toContainEqual(expect.objectContaining({ id: deploymentId, action: 'start', state: 'waiting' }));
|
||||
});
|
||||
|
||||
it('does not create a replacement when teardown fails or resume verification of the old instance', async () => {
|
||||
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoDestroyDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
const id = await queue('rebuild');
|
||||
await processLifecycle(); await processCreations(); await processLifecycle();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed' });
|
||||
expect(getDeployment(id)?.canResumeVerification).toBe(false);
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user