refactor: make provisioning own instance lifecycle operations
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
// Integration ticks reflect the production worker: callers submit/observe;
|
||||
// Provisioning alone executes jobs between those reconciliation passes.
|
||||
let campaignFlight: Promise<Awaited<ReturnType<typeof import('../../src/lib/server/pipeline/sweep.js').sweep>>> | null = null;
|
||||
export function sweep() {
|
||||
if (campaignFlight) return campaignFlight;
|
||||
campaignFlight = (async () => {
|
||||
const caller = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await caller.sweep(); await processProvisioning(); return caller.sweep();
|
||||
})().finally(() => { campaignFlight = null; });
|
||||
return campaignFlight;
|
||||
}
|
||||
let betaFlight: Promise<{ processed: number }> | null = null;
|
||||
export function betaTick() {
|
||||
if (betaFlight) return betaFlight;
|
||||
betaFlight = (async () => {
|
||||
const caller = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await caller.betaTick(); await processProvisioning(); return caller.betaTick();
|
||||
})().finally(() => { betaFlight = null; });
|
||||
return betaFlight;
|
||||
}
|
||||
let maintenanceFlight: Promise<void> | null = null;
|
||||
export function maintenanceTick(): Promise<void> {
|
||||
if (maintenanceFlight) return maintenanceFlight;
|
||||
maintenanceFlight = (async () => {
|
||||
const caller = await import('../../src/lib/server/fleet/plutonium.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await caller.maintenanceTick(); await processProvisioning(); await caller.maintenanceTick();
|
||||
})().finally(() => { maintenanceFlight = null; });
|
||||
return maintenanceFlight;
|
||||
}
|
||||
@@ -48,7 +48,7 @@ async function tick(seconds = 15, players = 4, healthy = true) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE servers SET last_checked_at = ?, last_seen_at = ?, last_players = ?, last_error = ?, container_state = 'running'")
|
||||
.run(now(), now(), players, healthy ? null : 'timeout');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
await betaTick();
|
||||
}
|
||||
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { maintenanceTick } from '../helpers/provisioning.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js';
|
||||
@@ -100,12 +101,31 @@ describe('maintenance timezones', () => {
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z');
|
||||
vi.setSystemTime(new Date('2026-03-07T11:00:00Z'));
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistent maintenance', () => {
|
||||
it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await m.maintenanceTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
const job = m.plutoniumStatus().jobs[0];
|
||||
expect(job.provisioning_id).toBeTruthy();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
// The source has not mirrored submission yet; cancellation must still
|
||||
// consult the authoritative operation, not its stale queued state.
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('queued');
|
||||
expect(m.cancelMaintenance(job.id)).toBe(false);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(Date.now() + 15_000); await processProvisioning();
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
});
|
||||
it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
expect(m.queueMaintenance([servers[0]])).toBe(1);
|
||||
@@ -113,31 +133,31 @@ describe('persistent maintenance', () => {
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
|
||||
await m.maintenanceTick(); await m.maintenanceTick();
|
||||
await maintenanceTick(); await maintenanceTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
await Promise.all([m.maintenanceTick(), m.maintenanceTick()]);
|
||||
await Promise.all([maintenanceTick(), maintenanceTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
|
||||
});
|
||||
it('shares rolling capacity across servers and survives closing the database', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
||||
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
|
||||
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
|
||||
});
|
||||
it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
||||
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); await m.maintenanceTick();
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -146,7 +166,7 @@ describe('persistent maintenance', () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
|
||||
await m.maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
|
||||
await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -156,7 +176,7 @@ describe('persistent maintenance', () => {
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
||||
m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
|
||||
});
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
|
||||
@@ -166,19 +186,22 @@ describe('persistent maintenance', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
|
||||
await m.maintenanceTick();
|
||||
await maintenanceTick();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(m.plutoniumStatus().jobs[0].detail).toContain('deployment or beta');
|
||||
expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
|
||||
});
|
||||
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
|
||||
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
|
||||
await m.maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
|
||||
await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs[0].state).toBe('done');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
it('validates API requests without changing the schedule or queue', async () => {
|
||||
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
|
||||
@@ -203,7 +226,7 @@ describe('persistent maintenance', () => {
|
||||
expect(m.plutoniumStatus().schedule.enabled).toBe(false);
|
||||
m.saveMaintenanceSchedule(true, '06:00', 'UTC');
|
||||
vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
|
||||
await m.maintenanceTick(); await m.maintenanceTick();
|
||||
await maintenanceTick(); await maintenanceTick();
|
||||
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
||||
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
|
||||
m.saveMaintenanceSchedule(false, '06:00', 'UTC');
|
||||
|
||||
@@ -71,7 +71,7 @@ async function beta(targets: string[], waitForHealth = true) {
|
||||
describe('rolling deployment restarts', () => {
|
||||
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect((await sweep()).restarted).toBe(1);
|
||||
@@ -98,7 +98,7 @@ describe('rolling deployment restarts', () => {
|
||||
await deployment('a-next', [servers[0]], 'drift');
|
||||
nextTick();
|
||||
await deployment('b-latest', [servers[1]], 'promote');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
@@ -111,7 +111,7 @@ describe('rolling deployment restarts', () => {
|
||||
await deployment();
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
@@ -119,7 +119,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep();
|
||||
@@ -138,7 +138,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let finish!: () => void;
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
@@ -164,7 +164,7 @@ describe('rolling deployment restarts', () => {
|
||||
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
await deployment('dep-2', [servers[0], servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -174,14 +174,18 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
await deployment('dep-2', [servers[0]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeNull();
|
||||
expect((await rows('dep-2'))[0].restart_state).toBe('pending');
|
||||
// Provisioning can submit the next job in the pass that verifies the
|
||||
// first, but it must issue a distinct restart and await its recovery.
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
@@ -189,12 +193,12 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('preserves occupied slots across app/module restarts', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const fresh = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -208,7 +212,7 @@ describe('rolling deployment restarts', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
@@ -216,7 +220,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
||||
await sweep();
|
||||
@@ -233,7 +237,7 @@ describe('rolling deployment restarts', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
||||
.run(servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
||||
@@ -242,7 +246,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('releases slots for definite local failures that could not have started a restart', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
||||
await sweep();
|
||||
@@ -254,7 +258,7 @@ describe('rolling deployment restarts', () => {
|
||||
describe('beta and deployment coordination', () => {
|
||||
it('limits beta restarts and releases capacity only after recovery', async () => {
|
||||
await beta(servers);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = betaTick();
|
||||
expect(betaTick()).toBe(first);
|
||||
@@ -268,8 +272,8 @@ describe('beta and deployment coordination', () => {
|
||||
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
||||
await beta([servers[1]]);
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -277,8 +281,8 @@ describe('beta and deployment coordination', () => {
|
||||
|
||||
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
||||
const { betaId } = await beta([servers[0]], false);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await betaTick();
|
||||
@@ -297,7 +301,7 @@ describe('beta and deployment coordination', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
||||
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
||||
@@ -306,6 +310,132 @@ describe('beta and deployment coordination', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provisioning ownership', () => {
|
||||
it('persists caller requests before probes and completes them without another caller tick', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep(); await sweep();
|
||||
const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
|
||||
const id = source.provisioning_id;
|
||||
expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
|
||||
expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0] }]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 });
|
||||
await processProvisioning(); nextTick(); await processProvisioning();
|
||||
expect(provisioningOperation(id)?.state).toBe('done');
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('pending');
|
||||
await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('lets concurrent callers submit while only the owner admits restart capacity', async () => {
|
||||
await deployment('campaign', [servers[0]]); await beta([servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
const first = processProvisioning();
|
||||
expect(processProvisioning()).toBe(first);
|
||||
await first;
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not replay an ambiguous campaign restart after recovery or app restart', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb(); vi.resetModules();
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
nextTick(); await fresh.sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('holds a paused beta request and cancels closed work without a beta runner tick', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' });
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
expect(requestLifecycle(servers[0], 'restart').ok).toBe(true);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
});
|
||||
|
||||
it('does not credit a legacy submission with an older completed recovery slot', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const old = new Date().toISOString(); nextTick();
|
||||
const submitted = new Date().toISOString();
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old);
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null });
|
||||
nextTick(); await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string };
|
||||
closeDb(); vi.resetModules();
|
||||
const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const source = { kind: 'beta' as const, id: String(betaId) };
|
||||
expect(requestRestart(servers[0], source).id).toBe(previous.id);
|
||||
releaseRestartRequest(servers[0], source);
|
||||
const next = requestRestart(servers[0], source);
|
||||
expect(next.id).not.toBe(previous.id);
|
||||
expect(provisioningOperation(previous.id)?.state).toBe('cancelled');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(provisioningOperation(next.id)?.state).toBe('awaiting-health');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout configuration', () => {
|
||||
it('honors a configured limit larger than one', async () => {
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -313,7 +443,7 @@ describe('rollout configuration', () => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
|
||||
@@ -6,7 +6,7 @@ let fixture: Fixture;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(['t5_zombie_theater_1', 't5_zombie_cosmodrome_1']);
|
||||
// No integrations → sweep marks rows manual, exercising that branch.
|
||||
// Provisioning must retain queued work while integrations are unset.
|
||||
delete process.env.KOMODO_ADDRESS;
|
||||
delete process.env.RCON_PASSWORD;
|
||||
});
|
||||
@@ -16,14 +16,12 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe('sweep', () => {
|
||||
it('marks all pending rows manual when integrations are unset', async () => {
|
||||
it('queues pending rows in Provisioning even when integrations are unset', async () => {
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await seedServerRegistry();
|
||||
|
||||
// Seed a deployment row directly — the code path being tested is the
|
||||
// "integrations not configured" branch, which just walks deployment_servers.
|
||||
const depId = 'dep-test-001';
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'test', ?, 'ui')")
|
||||
.run(depId, new Date().toISOString());
|
||||
@@ -39,12 +37,18 @@ describe('sweep', () => {
|
||||
.run(depId, 't5_zombie_cosmodrome_1');
|
||||
|
||||
await sweep();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
await sweep();
|
||||
|
||||
const rows = db()
|
||||
.prepare("SELECT restart_state FROM deployment_servers WHERE deployment_id = ?")
|
||||
.all(depId) as Array<{ restart_state: string }>;
|
||||
expect(rows.length).toBe(2);
|
||||
for (const r of rows) expect(r.restart_state).toBe('manual');
|
||||
for (const r of rows) expect(r.restart_state).toBe('pending');
|
||||
const operations = db().prepare('SELECT state, stage FROM server_deployments').all();
|
||||
expect(operations).toHaveLength(2);
|
||||
for (const operation of operations) expect(operation).toEqual({ state: 'waiting', stage: 'Waiting for Komodo and RCON configuration' });
|
||||
});
|
||||
|
||||
it('is a no-op when nothing is pending', async () => {
|
||||
|
||||
Reference in New Issue
Block a user