Fix native provisioning scheduling and progress reporting

2026-10-04 14:50:59 -04:00
parent 1bbee98b62
commit 2adadb0d05
18 changed files with 569 additions and 84 deletions
@@ -73,6 +73,129 @@ async function detail(id: string) {
 }

 describe('native instance provisioning', () => {
+    it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
+        const job = await queue();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
+        const queued = getDeployment(job.deploymentId)!;
+        expect(queued.progress?.phase).toBe('eligibility');
+        expect(queued.progress?.blocker).toBeNull();
+        expect(queued.backend).toBe('native');
+        await advance();
+        expect(getDeployment(job.deploymentId)!.progress?.recipeStatus).toBe('attempted');
+        await acknowledge(job.deploymentId);
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('done');
+        expect(isProvisioningDetail(result, job.deploymentId)).toBe(true);
+        expect(result.progress?.checks.map(check => check.key)).toEqual(['container', 'game', 'map', 'plutonium-release', 'plutonium-version']);
+        expect(result.progress?.checks.every(check => check.result === 'pass')).toBe(true);
+        expect(result.progress?.checks[0].source).toBe('agent');
+        const steps = provisioningGroups(result).flatMap(group => group.steps);
+        expect(steps.every(step => step.state === 'done')).toBe(true);
+        expect(steps.some(step => step.label.includes('Komodo'))).toBe(false);
+        expect(result.agentOperation?.state).toBe('completed');
+    });
+
+    it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) {
+            reportHostOperation(hostId, operation.id, { state, result: { state: 'running', healthy: true } });
+        }
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('running');
+        expect(result.agentOperation?.state).toBe('completed');
+        expect(result.events.filter(event => event.kind === 'agent')).toHaveLength(3);
+        expect(result.progress?.checkpoints.map(point => point.key)).toEqual(expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed']));
+        expect(result.progress?.command?.state).toBe('accepted');
+    });
+
+    it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => {
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const first = await queue(); await advance();
+        const operation = (await operationFor(first.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const second = await queue();
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = advance();
+        try {
+            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+            expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
+            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+        } finally {
+            release({ healthy: true, state: 'running' } as never);
+            await pending;
+        }
+    });
+
+    it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => {
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const first = await queue(); await advance();
+        const operation = (await operationFor(first.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning({ wait: false });
+        try {
+            const second = await queue();
+            await processProvisioning({ wait: false });
+            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
+        } finally {
+            release({ healthy: true, state: 'running' } as never);
+            await processProvisioning();
+        }
+    });
+
+    it('ignores a late health result after removal retires the operation', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = advance();
+        try {
+            await vi.waitFor(() => expect(release).toBeTypeOf('function'));
+            const { db } = await import('../../src/lib/server/db/index.js');
+            db().prepare("UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?")
+                .run(new Date().toISOString(), job.deploymentId);
+            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+            const before = getDeployment(job.deploymentId)!;
+            release({ healthy: true, state: 'running' } as never);
+            await pending;
+            const after = getDeployment(job.deploymentId)!;
+            expect(after.deployment.state).toBe('cancelled');
+            expect(after.progress).toEqual(before.progress);
+            expect(after.activity).toEqual(before.activity);
+        } finally { release({ healthy: true, state: 'running' } as never); await pending; }
+    });
+
+    it('records map mismatch evidence while retaining passed container and game checks', async () => {
+        const job = await queue(); await advance(); remote.map = 'zombie_moon';
+        await acknowledge(job.deploymentId);
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const result = getDeployment(job.deploymentId)!;
+        expect(result.deployment.state).toBe('awaiting-health');
+        expect(result.progress?.phase).toBe('verify-map');
+        expect(result.progress?.checks).toEqual(expect.arrayContaining([
+            expect.objectContaining({ key: 'container', result: 'pass' }),
+            expect.objectContaining({ key: 'game', result: 'pass' }),
+            expect.objectContaining({ key: 'map', result: 'waiting', actual: 'zombie_moon', expected: 'zombie_theater' })
+        ]));
+    });
+
    it('selects the latest game files automatically and rejects stale explicit selections', async () => {
        const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
        expect(() => nativeSelection({ hostId, image }, 't5')).toThrow('Upload T5 game files');