import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; const server = 't5_zombie_theater_1'; const campaign = 'held-rollback'; const before = Buffer.from('current live bytes'); const restored = Buffer.from('restored version'); const destination = () => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/rollback.gsc'); beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([server]); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); }); afterEach(async () => { await fixture.cleanup(); }); async function heldRollback(state = 'held') { const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); await writeFile(destination(), before); const now = new Date().toISOString(); db() .prepare( "INSERT INTO deployments (id, action, started_at, finished_at) VALUES (?, 'rollback', ?, ?)" ) .run(campaign, now, now); db() .prepare( `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, error) VALUES (?, ?, 1, ?, 'Waiting for manual activation')` ) .run(campaign, server, state); queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: 'rollback.gsc', content: restored, inventorySource: 'rollback:beta-1' }); } async function resume(id = campaign) { const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js'); return POST({ params: { id } }); } describe('campaign resume API', () => { it.each(['held', 'manual'])( 'queues %s rollback changes without touching live files', async (state) => { await heldRollback(state); const { db } = await import('../../src/lib/server/db/index.js'); const { kick } = await import('../../src/lib/server/pipeline/worker.js'); const response = await resume(); expect(response.status).toBe(202); expect(await response.json()).toEqual({ ok: true }); expect( db() .prepare( 'SELECT restart_state, error, provisioning_id FROM deployment_servers WHERE deployment_id = ?' ) .get(campaign) ).toEqual({ restart_state: 'pending', error: null, provisioning_id: null }); expect( db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign) ).toEqual({ finished_at: null }); expect( db() .prepare( 'SELECT state, (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_id = ?' ) .get(campaign) ).toEqual({ state: 'pending', content: restored }); expect(await readFile(destination())).toEqual(before); expect(kick).toHaveBeenCalledTimes(1); expect((await resume()).status).toBe(409); expect(kick).toHaveBeenCalledTimes(1); } ); it.each(['cancelled', 'failed', 'interrupted'])( 'releases an old %s operation so the worker creates fresh restart intent', async (state) => { await heldRollback(); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const { db } = await import('../../src/lib/server/db/index.js'); const old = requestRestart(server, { kind: 'campaign', id: campaign }); db() .prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?') .run(state, new Date().toISOString(), old.id); expect((await resume()).status).toBe(202); expect( db() .prepare( 'SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?' ) .get(campaign) ).toEqual({ provisioning_id: null }); const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); await sweep(); const next = requestRestart(server, { kind: 'campaign', id: campaign }); expect(next.id).not.toBe(old.id); expect(next).toMatchObject({ state: 'waiting', submitted_at: null }); expect( db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id) ).toEqual({ state }); expect( db() .prepare( 'SELECT operation_id FROM provisioning_sources WHERE source_id = ? ORDER BY rowid' ) .all(campaign) ).toEqual([{ operation_id: old.id }, { operation_id: next.id }]); expect(await readFile(destination())).toEqual(before); } ); it('upgrades queued file restarts without replaying submitted operations', async () => { await heldRollback('pending'); const { db } = await import('../../src/lib/server/db/index.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const job = requestRestart(server, { kind: 'campaign', id: campaign }); expect(job.action).toBe('rebuild'); db().prepare("UPDATE server_deployments SET action='restart' WHERE id=?").run(job.id); expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({ id: job.id, action: 'rebuild' }); db() .prepare( "UPDATE server_deployments SET action='restart',state='running',submitted_at=? WHERE id=?" ) .run(new Date().toISOString(), job.id); expect(requestRestart(server, { kind: 'campaign', id: campaign })).toMatchObject({ id: job.id, action: 'restart', state: 'running' }); }); it('retains an existing waiting operation when its held campaign resumes', async () => { await heldRollback(); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const old = requestRestart(server, { kind: 'campaign', id: campaign }); expect((await resume()).status).toBe(202); expect(requestRestart(server, { kind: 'campaign', id: campaign }).id).toBe(old.id); }); it.each(['missing', 'done', 'deleted'])( 'rejects a campaign with %s changes that cannot resume', async (condition) => { const { db } = await import('../../src/lib/server/db/index.js'); const { kick } = await import('../../src/lib/server/pipeline/worker.js'); if (condition !== 'missing') await heldRollback(condition === 'done' ? 'done' : 'held'); if (condition === 'deleted') { db().prepare('DELETE FROM servers WHERE name = ?').run(server); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); // Reusing its name must not resume old work. } const response = await resume(); expect(response.status).toBe(409); expect(await response.json()).toEqual({ error: 'This campaign has no held changes to resume.' }); expect(kick).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]); if (condition !== 'missing') { expect(await readFile(destination())).toEqual(before); expect( db() .prepare( 'SELECT restart_state FROM deployment_servers WHERE deployment_id = ?' ) .get(campaign) ).toEqual({ restart_state: condition === 'done' ? 'done' : 'held' }); expect( db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaign) ).toMatchObject({ finished_at: expect.any(String) }); } } ); });