import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; const server = 't5_zombie_theater_1'; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); async function recovering(step: 'restart' | 'rebuild') { const { host } = await prepareNativeHost([server]); const hosts = await import('../../src/lib/server/hosts/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); const { processNativeProvisioning, reconcileNativeProvisioningReceipt } = await import('../../src/lib/server/provisioning/native.js'); hosts.heartbeat(host.id, { ...hosts.getHost(host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] }); db().prepare('INSERT INTO servers(name) VALUES(?)').run(server); const instance = nativeInstance(server)!; hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'udp' }); hosts.reserveHostPort(host.id, instance.id, { port: instance.port, protocol: 'tcp' }); const deploymentId = beginDeployment(server, 'create'); const original = hosts.queueHostOperation( host.id, 'instance.create', {}, { instanceId: instance.id, generation: 1 } ); hosts.reportHostOperation(host.id, original.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } }); const recovery = hosts.queueHostOperation( host.id, `instance.${step}`, { unhealthyRecovery: true, ...(step === 'rebuild' ? { nextGeneration: 2 } : {}) }, { instanceId: instance.id, generation: 1 } ); db() .prepare( 'UPDATE native_provisioning SET operation_id=?,recovery_step=? WHERE deployment_id=?' ) .run(recovery.id, step, deploymentId); db() .prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?") .run(deploymentId); const commandFor = (id: string) => { const row = db() .prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?') .get(id) as { operation_id: string }; return hosts.getHostOperation(row.operation_id)!; }; return { host, instance, deploymentId, original, recovery, ...hosts, db, nativeInstance, cancelCreation, canCancelCreation, getDeployment, reconcileNativeProvisioningReceipt, commandFor, advance: () => processNativeProvisioning(() => null) }; } describe('cancelling creation during unhealthy recovery', () => { it('keeps the original creation identity and suppresses recovery restart replay', async () => { const api = await recovering('restart'); expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toContain( api.recovery.id ); const cleanup = api.cancelCreation(api.deploymentId)!; const removal = api.commandFor(cleanup); expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1, payload: { createOperationId: api.original.id } }); expect(removal.payload).toEqual({ createOperationId: api.original.id }); expect(api.pendingCreationCancellations(api.host.id)).toEqual([api.original.id]); expect( api .pendingHostOperations(api.host.id) .filter((operation) => operation.kind.startsWith('instance.')) .map((operation) => operation.id) ).toEqual([removal.id]); api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } }); await api.advance(); expect(api.getDeployment(cleanup)?.deployment.state).toBe('done'); expect(api.getHostOperation(api.recovery.id)?.state).toBe('failed'); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); }); it.each(['pending', 'running', 'unknown', 'completed'] as const)( 'cleans up both rebuild generations when recovery is %s', async (state) => { const api = await recovering('rebuild'); if (state !== 'pending') { api.pendingHostOperations(api.host.id); api.reportHostOperation(api.host.id, api.recovery.id, { state, result: { state: 'unhealthy', healthy: false } }); } if (state === 'completed') api.reconcileNativeProvisioningReceipt(api.deploymentId); const registeredGeneration = state === 'completed' ? 2 : 1; expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); expect(api.canCancelCreation(api.deploymentId)).toBe(true); const cleanup = api.cancelCreation(api.deploymentId)!; const removal = api.commandFor(cleanup); expect(removal).toMatchObject({ kind: 'instance.cancel-create', generation: 1, payload: { createOperationId: api.original.id, recoveryRebuildOperationId: api.recovery.id, recoveryGeneration: 2 } }); expect(api.pendingCreationCancellations(api.host.id)).toEqual([ api.original.id, api.recovery.id ]); expect( api .db() .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?') .get(cleanup) ).toEqual({ generation: registeredGeneration }); expect( api .pendingHostOperations(api.host.id) .filter((operation) => operation.kind.startsWith('instance.')) .map((operation) => operation.id) ).toEqual([removal.id]); // A delivered rebuild may report success after cancellation, while the // cleanup command already owns deletion of both possible containers. api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed', result: { state: 'unhealthy', healthy: false } }); api.reconcileNativeProvisioningReceipt(api.deploymentId); expect(api.nativeInstance(server)?.generation).toBe(registeredGeneration); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); api.reportHostOperation(api.host.id, removal.id, { state: 'completed', result: { exists: false } }); await api.advance(); expect(api.getDeployment(cleanup)?.deployment.state).toBe('done'); expect(api.getDeployment(api.deploymentId)?.deployment).toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) }); expect(api.nativeInstance(server)).toBeNull(); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); expect( api.db().prepare('SELECT name FROM servers WHERE name=?').get(server) ).toBeUndefined(); expect(api.pendingCreationCancellations(api.host.id)).toEqual([]); expect(api.cancelCreation(api.deploymentId)).toBe(cleanup); } ); it('does not adopt a changed generation without a completed recovery receipt', async () => { const api = await recovering('rebuild'); api.db() .prepare('UPDATE native_instances SET generation=2 WHERE id=?') .run(api.instance.id); expect(api.canCancelCreation(api.deploymentId)).toBe(false); expect(api.cancelCreation(api.deploymentId)).toBeNull(); api.reportHostOperation(api.host.id, api.recovery.id, { state: 'completed' }); api.db() .prepare('UPDATE native_instances SET generation=3 WHERE id=?') .run(api.instance.id); expect(api.cancelCreation(api.deploymentId)).toBeNull(); }); it('holds extended cleanup until the host supports recovery cancellation', async () => { const api = await recovering('rebuild'); const cleanup = api.cancelCreation(api.deploymentId)!; api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create'] }); expect(api.pendingHostOperations(api.host.id)).toEqual([]); await api.advance(); expect(api.getDeployment(cleanup)?.deployment.stage).toContain('Update the host agent'); expect(api.db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); api.heartbeat(api.host.id, { ...api.getHost(api.host.id)!.inventory, capabilities: ['instance.cancel-create', 'instance.unhealthy-recovery'] }); expect( api .pendingHostOperations(api.host.id) .filter((operation) => operation.kind.startsWith('instance.')) .map((operation) => operation.id) ).toEqual([api.commandFor(cleanup).id]); }); });