feat: add Watchdog recovery and drift rebuild workflows
This commit is contained in:
@@ -29,10 +29,13 @@ beforeEach(async () => {
|
||||
// Exercise the upgrade path, with history and references present before migration.
|
||||
const old = new Database(fixture.dbPath);
|
||||
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'"));
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
|
||||
.replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'")
|
||||
.replace("server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server TEXT NOT NULL REFERENCES servers(name),'));
|
||||
old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
|
||||
old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
|
||||
old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
|
||||
old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run();
|
||||
old.close();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
@@ -73,6 +76,7 @@ describe('provisioning operations', () => {
|
||||
expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
|
||||
expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
|
||||
expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
|
||||
expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
});
|
||||
@@ -244,3 +248,279 @@ describe('provisioning operations', () => {
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
async function incident(id = 'incident') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
||||
VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
|
||||
return id;
|
||||
}
|
||||
|
||||
async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { gameEvent } = await import('../helpers/telemetry.js');
|
||||
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
||||
ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`)
|
||||
.run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp }));
|
||||
}
|
||||
|
||||
describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
it('keeps the original running intent when a queued removal is cancelled', async () => {
|
||||
const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const removal = destroyServer(server);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal');
|
||||
expect(cancelRemoval(removal.deploymentId)).toBe(true);
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('preserves stop intent across process restarts, including an uncertain stop request', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const id = await queue('stop');
|
||||
finishDeployment(id, { ok: false, message: 'lost response' });
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
// Exercise a legacy upgrade where the stop exists but intent was not stored.
|
||||
db().prepare("UPDATE servers SET desired_state = 'running'").run();
|
||||
db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run();
|
||||
closeDb();
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
await incident();
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' });
|
||||
await queue('start');
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => {
|
||||
const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
|
||||
expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server }]);
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null });
|
||||
await incident(); await report(0);
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
await report(0);
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
|
||||
it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => {
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('restart');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; });
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 });
|
||||
await report(0);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 });
|
||||
});
|
||||
|
||||
it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => {
|
||||
const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
for (const offset of [-45_001, 1]) {
|
||||
await report(0, new Date(Date.now() + offset).toISOString());
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
expect(await probePlayerCount(server)).toBeNull();
|
||||
}
|
||||
await report(0);
|
||||
expect(telemetryEvidence(server, new Date().toISOString())).toBeNull();
|
||||
db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server);
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
});
|
||||
|
||||
it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning();
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild);
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
closeDb();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
removeServerKey(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId });
|
||||
saveServerKey(server, 'saved-key');
|
||||
db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 });
|
||||
});
|
||||
|
||||
it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
let reportHealthy!: (value: { ok: true; state: string }) => void;
|
||||
vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; }));
|
||||
const observation = refreshRestartRollout();
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
reportHealthy({ ok: true, state: 'running' });
|
||||
await observation;
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
});
|
||||
|
||||
it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => {
|
||||
const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
deploymentStage(rebuild.deploymentId, REMOVING_SERVER);
|
||||
finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' });
|
||||
db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run();
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
expect(resumeRebuild(rebuild.deploymentId)).toBe(true);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)")
|
||||
.run(rebuild.deploymentId, new Date().toISOString());
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run();
|
||||
const retry = requestWatchdogLifecycle(server, action, 'incident');
|
||||
if (!retry.ok) throw new Error(retry.error);
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: retry.deploymentId, healthy_at: null });
|
||||
await refreshRestartRollout();
|
||||
// Old process health cannot release capacity before this retry submits.
|
||||
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1);
|
||||
});
|
||||
|
||||
it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
const now = new Date().toISOString();
|
||||
db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now);
|
||||
db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId);
|
||||
db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server);
|
||||
db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.setSystemTime(Date.now() + 15_000); await report(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user