Persist beta, promotion, drift, and rollback payloads until occupancy and restart-capacity checks pass. Preserve rollback snapshots and campaign ordering, and allow held rollback campaigns to resume safely. Fixes #28 Validation: 805 tests passed, Svelte checks clean, production build passed.
113 lines
6.4 KiB
TypeScript
113 lines
6.4 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture;
|
|
const server = 't5_zombie_theater_1';
|
|
const campaign = 'held-rollback';
|
|
const before = Buffer.from('current live bytes');
|
|
const restored = Buffer.from('restored version');
|
|
const destination = () => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc');
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture([server]);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); });
|
|
|
|
async function heldRollback(state = 'held') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
await writeFile(destination(), before);
|
|
const now = new Date().toISOString();
|
|
db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)")
|
|
.run(campaign, now, now);
|
|
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error)
|
|
VALUES (?, ?, 1, ?, 'Waiting for manual activation')`).run(campaign, server, state);
|
|
queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: 'rollback.gsc',
|
|
content: restored, inventorySource: 'rollback:beta-1' });
|
|
}
|
|
|
|
async function resume(id = campaign) {
|
|
const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
|
|
return POST({ params: { id } });
|
|
}
|
|
|
|
describe('campaign resume API', () => {
|
|
it.each(['held', 'manual'])('queues %s rollback changes without touching live files', async (state) => {
|
|
await heldRollback(state);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
const response = await resume();
|
|
expect(response.status).toBe(202);
|
|
expect(await response.json()).toEqual({ ok: true });
|
|
expect(db().prepare('SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
|
|
.toEqual({ restart_state: 'pending', error: null, provisioning_id: null });
|
|
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)).toEqual({ finished_at: null });
|
|
expect(db().prepare('SELECT state, content FROM staged_file_changes WHERE source_id = ?').get(campaign))
|
|
.toEqual({ state: 'pending', content: restored });
|
|
expect(await readFile(destination())).toEqual(before);
|
|
expect(kick).toHaveBeenCalledTimes(1);
|
|
expect((await resume()).status).toBe(409);
|
|
expect(kick).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it.each(['cancelled', 'failed', 'interrupted'])('releases an old %s operation so the worker creates fresh restart intent', async (state) => {
|
|
await heldRollback();
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const old = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
db().prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
|
|
.run(state, new Date().toISOString(), old.id);
|
|
expect((await resume()).status).toBe(202);
|
|
expect(db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get(campaign))
|
|
.toEqual({ provisioning_id: null });
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
await sweep();
|
|
const next = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
expect(next.id).not.toBe(old.id);
|
|
expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)).toEqual({ state });
|
|
expect(db().prepare('SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid').all(campaign))
|
|
.toEqual([{ operation_id: old.id }, { operation_id: next.id }]);
|
|
expect(await readFile(destination())).toEqual(before);
|
|
});
|
|
|
|
it('retains an existing waiting operation when its held campaign resumes', async () => {
|
|
await heldRollback();
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const old = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
expect((await resume()).status).toBe(202);
|
|
expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id);
|
|
});
|
|
|
|
it.each(['missing', 'done', 'deleted'])('rejects a campaign with %s changes that cannot resume', async (condition) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held');
|
|
if (condition === 'deleted') {
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry(); // Reusing its name must not resume old work.
|
|
}
|
|
const response = await resume();
|
|
expect(response.status).toBe(409);
|
|
expect(await response.json()).toEqual({ error: 'This campaign has no held changes to resume.' });
|
|
expect(kick).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
|
|
if (condition !== 'missing') {
|
|
expect(await readFile(destination())).toEqual(before);
|
|
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaign))
|
|
.toEqual({ restart_state: condition === 'done' ? 'done' : 'held' });
|
|
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign))
|
|
.toMatchObject({ finished_at: expect.any(String) });
|
|
}
|
|
});
|
|
});
|