josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add cancellation and cleanup for native server creation

Browse Source
This commit is contained in:
josh
2026-10-04 18:00:11 -04:00
parent 19f1f4e279
commit 176b7dd86b
17 changed files with 482 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+114 -1
View File
@@ -9,7 +9,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
}));
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
let fixture: Fixture; let hostId: string; let libraryId: string;
const image = 'example.test/plutainer:test';
@@ -73,6 +73,119 @@ async function detail(id: string) {
}
describe('native instance provisioning', () => {
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
const job = await queue();
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
const response = POST({ params: { id: job.deploymentId } });
expect(response.status).toBe(200);
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
const next = await queue(); await advance(); await acknowledge(next.deploymentId);
expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
});
it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
const job = await queue();
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
const cleanup = cancelCreation(job.deploymentId)!;
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
expect((await detail(job.deploymentId)).state).toBe('cancelled');
expect((await detail(cleanup)).state).toBe('done');
expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
await advance();
expect(await operationFor(job.deploymentId)).toBeNull();
expect(await operationFor(cleanup)).toBeNull();
});
it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
const job = await queue(); await advance(); remote.players = 4;
const original = (await operationFor(job.deploymentId))!;
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
// A previously delivered command might still be executing when cancellation arrives.
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id);
const cleanup = cancelCreation(job.deploymentId)!;
const removal = (await operationFor(cleanup))!;
expect(removal.kind).toBe('instance.cancel-create');
expect(removal.payload).toEqual({ createOperationId: original.id });
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id);
expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id);
await advance();
expect(nativeRequest).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
await advance();
expect((await detail(job.deploymentId)).state).toBe('cancelled');
expect(nativeRequest).not.toHaveBeenCalled();
reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
await advance();
expect((await detail(cleanup)).state).toBe('done');
expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) });
});
it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
const job = await queue(); await advance();
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
const cleanup = cancelCreation(job.deploymentId)!;
expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]);
await advance();
expect((await detail(cleanup)).stage).toContain('Update the host agent');
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId);
await advance();
expect((await detail(cleanup)).stage).toContain('unavailable');
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1);
await acknowledge(cleanup);
expect((await detail(cleanup)).state).toBe('done');
});
it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
const job = await queue(); await advance();
const operation = (await operationFor(job.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
let release!: (value: never) => void;
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
const pending = advance();
try {
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const waitingRemoval = destroyServer(job.name).deploymentId!;
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const cleanup = cancelCreation(job.deploymentId)!;
expect(cleanup).toBeTruthy();
expect((await detail(waitingRemoval)).state).toBe('cancelled');
release({ healthy: true, state: 'running' } as never);
await pending;
expect((await detail(job.deploymentId)).state).toBe('cancelled');
expect((await detail(cleanup)).state).toBe('running');
} finally { release({ healthy: true, state: 'running' } as never); await pending; }
});
it('does not offer creation cancellation after success or against a replacement instance', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(canCancelCreation(job.deploymentId)).toBe(false);
expect(cancelCreation(job.deploymentId)).toBeNull();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId);
db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
expect(cancelCreation(job.deploymentId)).toBeNull();
});
it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
const job = await queue();
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
tests/unit/provisioning-presentation.test.ts
+11
View File
@@ -19,6 +19,17 @@ function fixture(): ProvisioningDetail {
}
describe('provisioning outcome and evidence presentation', () => {
it('distinguishes cancellation from confirmed cleanup and validates its link data', () => {
const detail = fixture();
detail.deployment.state = 'cancelled';
detail.canCancelCreation = false;
detail.cancellation = { cleanupId: 'cleanup-a', state: 'running', stage: 'Waiting for host', error: null };
expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
expect(provisioningOutcome(detail).description).toContain('ports remain reserved');
detail.cancellation.state = 'done';
expect(provisioningOutcome(detail).description).toContain('have been removed');
expect(isProvisioningDetail({ ...detail, cancellation: { state: 'done' } }, detail.deployment.id)).toBe(false);
});
it.each(provisioningActions)('describes cancelled %s as terminal, with no automatic execution claim', action => {
const detail = fixture();
detail.deployment.action = action;