197 lines
8.6 KiB
TypeScript
197 lines
8.6 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture;
|
|
const server = 't5_zombie_theater_1';
|
|
const campaign = 'held-rollback';
|
|
const before = Buffer.from('current live bytes');
|
|
const restored = Buffer.from('restored version');
|
|
const destination = () =>
|
|
join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc');
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture([server]);
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
async function heldRollback(state = 'held') {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
await writeFile(destination(), before);
|
|
const now = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)"
|
|
)
|
|
.run(campaign, now, now);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error)
|
|
VALUES (?, ?, 1, ?, 'Waiting for manual activation')`
|
|
)
|
|
.run(campaign, server, state);
|
|
queueFileChange({
|
|
sourceKind: 'campaign',
|
|
sourceId: campaign,
|
|
server,
|
|
kind: 'script',
|
|
relPath: 'rollback.gsc',
|
|
content: restored,
|
|
inventorySource: 'rollback:beta-1'
|
|
});
|
|
}
|
|
|
|
async function resume(id = campaign) {
|
|
const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
|
|
return POST({ params: { id } });
|
|
}
|
|
|
|
describe('campaign resume API', () => {
|
|
it.each(['held', 'manual'])(
|
|
'queues %s rollback changes without touching live files',
|
|
async (state) => {
|
|
await heldRollback(state);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
const response = await resume();
|
|
expect(response.status).toBe(202);
|
|
expect(await response.json()).toEqual({ ok: true });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?'
|
|
)
|
|
.get(campaign)
|
|
).toEqual({ restart_state: 'pending', error: null, provisioning_id: null });
|
|
expect(
|
|
db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)
|
|
).toEqual({ finished_at: null });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT state, (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_id = ?'
|
|
)
|
|
.get(campaign)
|
|
).toEqual({ state: 'pending', content: restored });
|
|
expect(await readFile(destination())).toEqual(before);
|
|
expect(kick).toHaveBeenCalledTimes(1);
|
|
expect((await resume()).status).toBe(409);
|
|
expect(kick).toHaveBeenCalledTimes(1);
|
|
}
|
|
);
|
|
|
|
it.each(['cancelled', 'failed', 'interrupted'])(
|
|
'releases an old %s operation so the worker creates fresh restart intent',
|
|
async (state) => {
|
|
await heldRollback();
|
|
const { requestRestart } =
|
|
await import('../../src/lib/server/provisioning/requests.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const old = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
db()
|
|
.prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
|
|
.run(state, new Date().toISOString(), old.id);
|
|
expect((await resume()).status).toBe(202);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?'
|
|
)
|
|
.get(campaign)
|
|
).toEqual({ provisioning_id: null });
|
|
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
|
await sweep();
|
|
const next = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
expect(next.id).not.toBe(old.id);
|
|
expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
|
|
expect(
|
|
db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)
|
|
).toEqual({ state });
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid'
|
|
)
|
|
.all(campaign)
|
|
).toEqual([{ operation_id: old.id }, { operation_id: next.id }]);
|
|
expect(await readFile(destination())).toEqual(before);
|
|
}
|
|
);
|
|
|
|
it('upgrades queued file restarts without replaying submitted operations', async () => {
|
|
await heldRollback('pending');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const job = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
expect(job.action).toBe('rebuild');
|
|
db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(job.id);
|
|
expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
|
|
id: job.id,
|
|
action: 'rebuild'
|
|
});
|
|
db()
|
|
.prepare(
|
|
"UPDATE server_deployments SET action='restart',state='running',submitted_at=? WHERE id=?"
|
|
)
|
|
.run(new Date().toISOString(), job.id);
|
|
expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({
|
|
id: job.id,
|
|
action: 'restart',
|
|
state: 'running'
|
|
});
|
|
});
|
|
|
|
it('retains an existing waiting operation when its held campaign resumes', async () => {
|
|
await heldRollback();
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const old = requestRestart(server, { kind: 'campaign', id: campaign });
|
|
expect((await resume()).status).toBe(202);
|
|
expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id);
|
|
});
|
|
|
|
it.each(['missing', 'done', 'deleted'])(
|
|
'rejects a campaign with %s changes that cannot resume',
|
|
async (condition) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held');
|
|
if (condition === 'deleted') {
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
|
const { seedServerRegistry } =
|
|
await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry(); // Reusing its name must not resume old work.
|
|
}
|
|
const response = await resume();
|
|
expect(response.status).toBe(409);
|
|
expect(await response.json()).toEqual({
|
|
error: 'This campaign has no held changes to resume.'
|
|
});
|
|
expect(kick).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
|
|
if (condition !== 'missing') {
|
|
expect(await readFile(destination())).toEqual(before);
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT restart_state FROM deployment_servers WHERE deployment_id = ?'
|
|
)
|
|
.get(campaign)
|
|
).toEqual({ restart_state: condition === 'done' ? 'done' : 'held' });
|
|
expect(
|
|
db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign)
|
|
).toMatchObject({ finished_at: expect.any(String) });
|
|
}
|
|
}
|
|
);
|
|
});
|