562 lines
40 KiB
TypeScript
562 lines
40 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import Database from 'better-sqlite3';
|
|
import { readFile, writeFile, access } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
|
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
|
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
|
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture;
|
|
const server = 't5_zombie_theater_2';
|
|
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
|
fixture = await newFixture([server]);
|
|
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
|
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
|
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
|
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
|
// Exercise the upgrade path, with history and references present before migration.
|
|
const old = new Database(fixture.dbPath);
|
|
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
|
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
|
|
.replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'")
|
|
.replace("server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server TEXT NOT NULL REFERENCES servers(name),'));
|
|
old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
|
|
old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
|
|
old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
|
|
old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run();
|
|
old.close();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
|
VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
|
|
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
saveServerKey(server, 'saved-key');
|
|
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
|
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
vi.mocked(kick).mockResolvedValue(undefined);
|
|
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
|
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
|
|
|
async function operation(id: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id);
|
|
}
|
|
|
|
async function queue(action: 'start' | 'restart' | 'stop' | 'rebuild') {
|
|
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const result = requestLifecycle(server, action);
|
|
expect(result.ok).toBe(true);
|
|
if (!result.ok) throw new Error(result.error);
|
|
return result.deploymentId;
|
|
}
|
|
|
|
describe('provisioning operations', () => {
|
|
it('migrates the action constraint without losing row ordering, activity, or foreign keys', async () => {
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
await queue('start');
|
|
closeDb();
|
|
expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
|
|
expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
|
|
expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
|
|
expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' });
|
|
expect(db().pragma('foreign_key_check')).toEqual([]);
|
|
expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
|
|
});
|
|
|
|
it('starts without an RCON empty check and verifies recovery after the grace period', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const id = await queue('start');
|
|
await processLifecycle();
|
|
expect(komodoRuntimeAction).toHaveBeenCalledWith(server, 'start');
|
|
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
|
expect(await operation(id)).toMatchObject({ action: 'start', state: 'awaiting-health' });
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
|
vi.setSystemTime(Date.now() + 15_000);
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'done' });
|
|
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('deduplicates requests, rejects conflicting actions, and waits for a fresh empty reading', async () => {
|
|
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const id = await queue('restart');
|
|
expect(requestLifecycle(server, 'restart')).toEqual({ ok: true, deploymentId: id });
|
|
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
|
|
expect(destroyServer(server)).toMatchObject({ ok: false, kind: 'conflict' });
|
|
for (const players of [4, null]) {
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'waiting' });
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
}
|
|
await processLifecycle();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('does not repeat an uncertain restart and retains its recovery slot', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
|
const id = await queue('restart');
|
|
await processLifecycle(); await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'failed', error: 'komodo: timeout' });
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
|
});
|
|
|
|
it('keeps stop pending on unknown health and completes only on a confirmed stopped state', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { komodoInspectHealthy, komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const id = await queue('stop');
|
|
await processLifecycle();
|
|
closeDb();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'network error' });
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, state: 'exited', reason: 'stopped' });
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'done' });
|
|
expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
|
|
await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
|
|
});
|
|
|
|
it('rebuilds from nothing in the same slot and port, then verifies the fresh instance', async () => {
|
|
const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { processCreations, nextLobbyNumber, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const sentinel = join(fixture.t5, server, 'old-instance.dat');
|
|
await writeFile(sentinel, 'must not survive');
|
|
const id = await queue('rebuild');
|
|
// Promotions after queueing must be included in the fresh instance.
|
|
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { script } = createScript({ name: 'Fresh promoted script', targetRelPath: 'fresh.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('fresh content') });
|
|
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
|
await processLifecycle();
|
|
expect(komodoDestroyDeployment).toHaveBeenCalledWith(server);
|
|
expect(komodoDeleteDeployment).toHaveBeenCalledWith(server);
|
|
await expect(access(sentinel)).rejects.toThrow();
|
|
expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
|
expect(await operation(id)).toMatchObject({ state: 'queued', action: 'rebuild' });
|
|
expect(nextAvailablePort()).toBe(28961);
|
|
expect(nextLobbyNumber('t5_zombie_theater', ['t5_zombie_theater_1'])).toBe(3);
|
|
expect(requestLifecycle(server, 'rebuild')).toEqual({ ok: true, deploymentId: id });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledWith(expect.objectContaining({ name: server, ports: ['28960:28960/udp', '28960:28960/tcp'] }));
|
|
await expect(access(sentinel)).rejects.toThrow();
|
|
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
|
expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content');
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.setSystemTime(Date.now() + 1);
|
|
await processCreations();
|
|
expect(await operation(id)).toMatchObject({ state: 'done' });
|
|
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
expect(runningVersions().find(version => version.server === server)).toMatchObject({ revision: 5354, update_status: 'current' });
|
|
});
|
|
|
|
it('rejects a rebuild without a saved key before touching the existing instance', async () => {
|
|
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
removeServerKey(server);
|
|
expect(requestLifecycle(server, 'rebuild')).toMatchObject({ ok: false, error: expect.stringContaining('Save a server key') });
|
|
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retains the rebuild cleanup checkpoint and reservation while waiting for capacity', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { beginDeployment, deploymentStage, finishDeployment, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const id = await queue('rebuild');
|
|
// Model a crash after confirmed remote deletion and registry cleanup.
|
|
deploymentStage(id, REMOVING_SERVER);
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
|
finishDeployment(id, { ok: false, message: 'cleanup interrupted' });
|
|
expect(nextAvailablePort()).toBe(28961);
|
|
expect(resumeRebuild(id)).toBe(true);
|
|
const occupied = beginDeployment('another-server', 'create');
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
|
|
closeDb();
|
|
finishDeployment(occupied, { ok: true });
|
|
await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'queued' });
|
|
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('allows explicit destruction of a stalled startup and records the interrupted verification', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const id = await queue('start');
|
|
await processLifecycle();
|
|
expect(destroyServer(server)).toMatchObject({ ok: true });
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
|
await processDestructions();
|
|
expect(await operation(id)).toMatchObject({ state: 'cancelled', error: 'Server deleted', server_deleted_at: expect.any(String) });
|
|
});
|
|
|
|
it('validates lifecycle API input and returns the durable operation ID', async () => {
|
|
const { POST, GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const submit = (body: unknown) => POST({ request: new Request('http://localhost/api/provisioning', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
|
|
}) });
|
|
for (const body of [null, { server, action: 'erase' }, { server: 5, action: 'start' }]) expect((await submit(body)).status).toBe(400);
|
|
const result = await submit({ server, action: 'start' });
|
|
expect(result.status).toBe(202);
|
|
const { deploymentId } = await result.json();
|
|
expect((await GET({ url: new URL('http://localhost/api/provisioning') }).json()).deployments).toContainEqual(expect.objectContaining({ id: deploymentId, action: 'start', state: 'waiting' }));
|
|
});
|
|
|
|
it('does not create a replacement when teardown fails or resume verification of the old instance', async () => {
|
|
const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { komodoDestroyDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
|
const id = await queue('rebuild');
|
|
await processLifecycle(); await processCreations(); await processLifecycle();
|
|
expect(await operation(id)).toMatchObject({ state: 'failed' });
|
|
expect(getDeployment(id)?.canResumeVerification).toBe(false);
|
|
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
async function incident(id = 'incident') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
// These lifecycle tests exercise an actual container failure. Healthy
|
|
// containers with fresh game evidence now cancel obsolete health work.
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' });
|
|
const now = new Date().toISOString();
|
|
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
|
VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
|
|
return id;
|
|
}
|
|
|
|
async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { gameEvent } = await import('../helpers/telemetry.js');
|
|
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
|
ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`)
|
|
.run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp }));
|
|
}
|
|
|
|
describe('Watchdog provisioning and telemetry evidence', () => {
|
|
it('keeps the original running intent when a queued removal is cancelled', async () => {
|
|
const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const removal = destroyServer(server);
|
|
if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal');
|
|
expect(cancelRemoval(removal.deploymentId)).toBe(true);
|
|
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
|
});
|
|
|
|
it('preserves stop intent across process restarts, including an uncertain stop request', async () => {
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const id = await queue('stop');
|
|
finishDeployment(id, { ok: false, message: 'lost response' });
|
|
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
|
// Exercise a legacy upgrade where the stop exists but intent was not stored.
|
|
db().prepare("UPDATE servers SET desired_state = 'running'").run();
|
|
db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run();
|
|
closeDb();
|
|
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
|
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
|
await incident();
|
|
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' });
|
|
await queue('start');
|
|
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
|
});
|
|
|
|
it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => {
|
|
const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
expect(result.ok).toBe(true);
|
|
if (!result.ok) throw new Error(result.error);
|
|
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
|
|
expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server,
|
|
label: 'Server down', href: '/maintenance/incident' }]);
|
|
expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
|
|
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
|
await processProvisioning();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run();
|
|
await processProvisioning();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => {
|
|
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
|
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null });
|
|
await incident(); await report(0);
|
|
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
if (!result.ok) throw new Error(result.error);
|
|
await processProvisioning();
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.setSystemTime(Date.now() + 15_000);
|
|
await processProvisioning();
|
|
expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
|
await report(0);
|
|
vi.setSystemTime(Date.now() + 15_000);
|
|
await processProvisioning();
|
|
expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' });
|
|
});
|
|
|
|
it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => {
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const id = await queue('restart');
|
|
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; });
|
|
await processProvisioning();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 });
|
|
await report(0);
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3);
|
|
await processProvisioning();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 });
|
|
});
|
|
|
|
it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => {
|
|
const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js');
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
|
for (const offset of [-90_001, 1]) {
|
|
await report(0, new Date(Date.now() + offset).toISOString());
|
|
expect(telemetryEvidence(server)).toBeNull();
|
|
expect(await probePlayerCount(server)).toBeNull();
|
|
}
|
|
await report(0);
|
|
expect(telemetryEvidence(server, new Date().toISOString())).toBeNull();
|
|
db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server);
|
|
expect(telemetryEvidence(server)).toBeNull();
|
|
});
|
|
|
|
it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => {
|
|
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await processProvisioning();
|
|
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
|
vi.setSystemTime(Date.now() + 300_000);
|
|
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
|
if (!rebuild.ok) throw new Error(rebuild.error);
|
|
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild);
|
|
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
|
expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
|
closeDb();
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
|
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
|
await processProvisioning();
|
|
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
|
await processProvisioning();
|
|
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
|
expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined();
|
|
});
|
|
|
|
it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => {
|
|
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
|
removeServerKey(server);
|
|
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
|
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
|
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId });
|
|
saveServerKey(server, 'saved-key');
|
|
db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server);
|
|
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
|
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' });
|
|
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 });
|
|
});
|
|
|
|
it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => {
|
|
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
|
let reportHealthy!: (value: { ok: true; state: string }) => void;
|
|
vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; }));
|
|
const observation = refreshRestartRollout();
|
|
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
|
if (!rebuild.ok) throw new Error(rebuild.error);
|
|
reportHealthy({ ok: true, state: 'running' });
|
|
await observation;
|
|
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
|
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
|
});
|
|
|
|
it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => {
|
|
const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
|
if (!rebuild.ok) throw new Error(rebuild.error);
|
|
deploymentStage(rebuild.deploymentId, REMOVING_SERVER);
|
|
finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' });
|
|
db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run();
|
|
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
|
expect(resumeRebuild(rebuild.deploymentId)).toBe(true);
|
|
await processProvisioning();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
|
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => {
|
|
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
|
const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
|
if (!restart.ok) throw new Error(restart.error);
|
|
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
|
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
|
if (!rebuild.ok) throw new Error(rebuild.error);
|
|
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)")
|
|
.run(rebuild.deploymentId, new Date().toISOString());
|
|
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' });
|
|
await processProvisioning();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' });
|
|
db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run();
|
|
const retry = requestWatchdogLifecycle(server, action, 'incident');
|
|
if (!retry.ok) throw new Error(retry.error);
|
|
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
|
.toEqual({ operation_id: retry.deploymentId, healthy_at: null });
|
|
await refreshRestartRollout();
|
|
// Old process health cannot release capacity before this retry submits.
|
|
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
|
if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
|
await processProvisioning();
|
|
expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
|
expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1);
|
|
});
|
|
|
|
it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => {
|
|
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await incident();
|
|
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
|
if (!rebuild.ok) throw new Error(rebuild.error);
|
|
const now = new Date().toISOString();
|
|
db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now);
|
|
db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId);
|
|
db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server);
|
|
db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server);
|
|
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
|
await processProvisioning();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
|
expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId });
|
|
expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 });
|
|
expect(db().pragma('foreign_key_check')).toEqual([]);
|
|
await processProvisioning();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
|
vi.setSystemTime(Date.now() + 15_000); await report(0);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
|
await processProvisioning();
|
|
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
|
|
});
|
|
});
|
|
|
|
describe('cancelling submitted legacy commands', () => {
|
|
it.each(['start', 'restart', 'stop'] as const)('ignores a late %s response after cancellation', async action => {
|
|
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
let release!: (value: { ok: true }) => void;
|
|
const command = action === 'restart' ? komodo.komodoRestart : komodo.komodoRuntimeAction;
|
|
vi.mocked(command).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
|
const id = await queue(action);
|
|
const pending = processProvisioning();
|
|
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
release({ ok: true });
|
|
await pending;
|
|
expect(await operation(id)).toMatchObject({ state: 'cancelled', stage: 'Cancelled by operator' });
|
|
expect(db().prepare('SELECT * FROM restart_rollout WHERE operation_id=?').all(id)).toEqual([]);
|
|
const replacement = await queue(action);
|
|
await processProvisioning();
|
|
expect(await operation(replacement)).toMatchObject({ state: 'awaiting-health' });
|
|
expect(command).toHaveBeenCalledTimes(2);
|
|
});
|
|
});
|