Add cancellation and cleanup for native server creation
This commit is contained in:
@@ -9,7 +9,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
const image = 'example.test/plutainer:test';
|
||||
@@ -73,6 +73,119 @@ async function detail(id: string) {
|
||||
}
|
||||
|
||||
describe('native instance provisioning', () => {
|
||||
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
|
||||
const job = await queue();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
||||
const response = POST({ params: { id: job.deploymentId } });
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
|
||||
const next = await queue(); await advance(); await acknowledge(next.deploymentId);
|
||||
expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
|
||||
expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
|
||||
});
|
||||
it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
|
||||
const job = await queue();
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await detail(cleanup)).state).toBe('done');
|
||||
expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
await advance();
|
||||
expect(await operationFor(job.deploymentId)).toBeNull();
|
||||
expect(await operationFor(cleanup)).toBeNull();
|
||||
});
|
||||
|
||||
it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
|
||||
const job = await queue(); await advance(); remote.players = 4;
|
||||
const original = (await operationFor(job.deploymentId))!;
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
||||
// A previously delivered command might still be executing when cancellation arrives.
|
||||
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id);
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
const removal = (await operationFor(cleanup))!;
|
||||
expect(removal.kind).toBe('instance.cancel-create');
|
||||
expect(removal.payload).toEqual({ createOperationId: original.id });
|
||||
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
||||
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id);
|
||||
expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id);
|
||||
await advance();
|
||||
expect(nativeRequest).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
||||
reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
|
||||
await advance();
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect(nativeRequest).not.toHaveBeenCalled();
|
||||
reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
|
||||
await advance();
|
||||
expect((await detail(cleanup)).state).toBe('done');
|
||||
expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
||||
expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
|
||||
const job = await queue(); await advance();
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]);
|
||||
await advance();
|
||||
expect((await detail(cleanup)).stage).toContain('Update the host agent');
|
||||
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId);
|
||||
await advance();
|
||||
expect((await detail(cleanup)).stage).toContain('unavailable');
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
||||
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
||||
expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1);
|
||||
await acknowledge(cleanup);
|
||||
expect((await detail(cleanup)).state).toBe('done');
|
||||
});
|
||||
|
||||
it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
|
||||
const job = await queue(); await advance();
|
||||
const operation = (await operationFor(job.deploymentId))!;
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
|
||||
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
||||
let release!: (value: never) => void;
|
||||
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
const pending = advance();
|
||||
try {
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const waitingRemoval = destroyServer(job.name).deploymentId!;
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect(cleanup).toBeTruthy();
|
||||
expect((await detail(waitingRemoval)).state).toBe('cancelled');
|
||||
release({ healthy: true, state: 'running' } as never);
|
||||
await pending;
|
||||
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
||||
expect((await detail(cleanup)).state).toBe('running');
|
||||
} finally { release({ healthy: true, state: 'running' } as never); await pending; }
|
||||
});
|
||||
|
||||
it('does not offer creation cancellation after success or against a replacement instance', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
expect(canCancelCreation(job.deploymentId)).toBe(false);
|
||||
expect(cancelCreation(job.deploymentId)).toBeNull();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId);
|
||||
db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
|
||||
expect(cancelCreation(job.deploymentId)).toBeNull();
|
||||
});
|
||||
|
||||
it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
|
||||
const job = await queue();
|
||||
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
|
||||
Reference in New Issue
Block a user