refactor: make provisioning own instance lifecycle operations
This commit is contained in:
@@ -71,7 +71,7 @@ async function beta(targets: string[], waitForHealth = true) {
|
||||
describe('rolling deployment restarts', () => {
|
||||
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect((await sweep()).restarted).toBe(1);
|
||||
@@ -98,7 +98,7 @@ describe('rolling deployment restarts', () => {
|
||||
await deployment('a-next', [servers[0]], 'drift');
|
||||
nextTick();
|
||||
await deployment('b-latest', [servers[1]], 'promote');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
@@ -111,7 +111,7 @@ describe('rolling deployment restarts', () => {
|
||||
await deployment();
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
@@ -119,7 +119,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep();
|
||||
@@ -138,7 +138,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let finish!: () => void;
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
@@ -164,7 +164,7 @@ describe('rolling deployment restarts', () => {
|
||||
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
await deployment('dep-2', [servers[0], servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -174,14 +174,18 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
await deployment('dep-2', [servers[0]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeNull();
|
||||
expect((await rows('dep-2'))[0].restart_state).toBe('pending');
|
||||
// Provisioning can submit the next job in the pass that verifies the
|
||||
// first, but it must issue a distinct restart and await its recovery.
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
@@ -189,12 +193,12 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('preserves occupied slots across app/module restarts', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const fresh = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -208,7 +212,7 @@ describe('rolling deployment restarts', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
@@ -216,7 +220,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
||||
await sweep();
|
||||
@@ -233,7 +237,7 @@ describe('rolling deployment restarts', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
||||
.run(servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
||||
@@ -242,7 +246,7 @@ describe('rolling deployment restarts', () => {
|
||||
|
||||
it('releases slots for definite local failures that could not have started a restart', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
||||
await sweep();
|
||||
@@ -254,7 +258,7 @@ describe('rolling deployment restarts', () => {
|
||||
describe('beta and deployment coordination', () => {
|
||||
it('limits beta restarts and releases capacity only after recovery', async () => {
|
||||
await beta(servers);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = betaTick();
|
||||
expect(betaTick()).toBe(first);
|
||||
@@ -268,8 +272,8 @@ describe('beta and deployment coordination', () => {
|
||||
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
||||
await beta([servers[1]]);
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
@@ -277,8 +281,8 @@ describe('beta and deployment coordination', () => {
|
||||
|
||||
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
||||
const { betaId } = await beta([servers[0]], false);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await betaTick();
|
||||
@@ -297,7 +301,7 @@ describe('beta and deployment coordination', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
||||
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { betaTick } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
||||
@@ -306,6 +310,132 @@ describe('beta and deployment coordination', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provisioning ownership', () => {
|
||||
it('persists caller requests before probes and completes them without another caller tick', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep(); await sweep();
|
||||
const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
|
||||
const id = source.provisioning_id;
|
||||
expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
|
||||
expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0] }]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 });
|
||||
await processProvisioning(); nextTick(); await processProvisioning();
|
||||
expect(provisioningOperation(id)?.state).toBe('done');
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('pending');
|
||||
await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('lets concurrent callers submit while only the owner admits restart capacity', async () => {
|
||||
await deployment('campaign', [servers[0]]); await beta([servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
const first = processProvisioning();
|
||||
expect(processProvisioning()).toBe(first);
|
||||
await first;
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not replay an ambiguous campaign restart after recovery or app restart', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb(); vi.resetModules();
|
||||
const fresh = await import('../helpers/provisioning.js');
|
||||
nextTick(); await fresh.sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('holds a paused beta request and cancels closed work without a beta runner tick', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' });
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
|
||||
await processProvisioning();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' });
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
|
||||
await sweep(); nextTick(); await sweep();
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
expect(requestLifecycle(servers[0], 'restart').ok).toBe(true);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('failed');
|
||||
});
|
||||
|
||||
it('does not credit a legacy submission with an older completed recovery slot', async () => {
|
||||
await deployment('campaign', [servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const old = new Date().toISOString(); nextTick();
|
||||
const submitted = new Date().toISOString();
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old);
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted);
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null });
|
||||
nextTick(); await sweep();
|
||||
expect((await rows('campaign'))[0].restart_state).toBe('done');
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
await betaTick();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string };
|
||||
closeDb(); vi.resetModules();
|
||||
const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const source = { kind: 'beta' as const, id: String(betaId) };
|
||||
expect(requestRestart(servers[0], source).id).toBe(previous.id);
|
||||
releaseRestartRequest(servers[0], source);
|
||||
const next = requestRestart(servers[0], source);
|
||||
expect(next.id).not.toBe(previous.id);
|
||||
expect(provisioningOperation(previous.id)?.state).toBe('cancelled');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(provisioningOperation(next.id)?.state).toBe('awaiting-health');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout configuration', () => {
|
||||
it('honors a configured limit larger than one', async () => {
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -313,7 +443,7 @@ describe('rollout configuration', () => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { sweep } = await import('../helpers/provisioning.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
|
||||
Reference in New Issue
Block a user