Richard Clark
5f0fc86cf9
Core tests / core (push) Successful in 9m57s
Full verification / application (push) Failing after 5m16s
Full verification / local-brain (push) Failing after 6m15s
Realm simulation / simulated-live-world (push) Failing after 5m32s
WoW integration lab / real-worldserver (push) Canceled after 17m23s
Source branch: feature/ai-brain-router Source commit: c2a09e0460465201a58aaab56c71424e9a96405e
36 lines
1.8 KiB
JavaScript
36 lines
1.8 KiB
JavaScript
import {readFile} from 'node:fs/promises';
|
|
|
|
const base=(process.env.OLLAMA_URL||'http://127.0.0.1:11434').replace(/\/$/,'');
|
|
const models=(process.env.MODELS||'qwen3.5:4b').split(',').map(x=>x.trim()).filter(Boolean);
|
|
const scenarios=JSON.parse(await readFile(new URL('../benchmarks/brain-scenarios.json',import.meta.url),'utf8'));
|
|
|
|
function prompt(s){
|
|
return [
|
|
{role:'system',content:'You choose one high-level intention for a person living in Azeroth. Stay in character. Return only JSON {"action":"explore|quest|socialize|trade|rest","reason":"brief in-character rationale","goal":"short goal"}. Use only an allowed action.'},
|
|
{role:'user',content:JSON.stringify(s)}
|
|
];
|
|
}
|
|
|
|
const rows=[];
|
|
for(const model of models){
|
|
for(const scenario of scenarios){
|
|
const started=performance.now();
|
|
let valid=false,allowed=false,action='',reason='',error='';
|
|
try{
|
|
const response=await fetch(base+'/api/chat',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({model,messages:prompt(scenario),stream:false,format:'json',options:{temperature:0.7,num_predict:260}})});
|
|
if(!response.ok)throw new Error('HTTP '+response.status);
|
|
const data=await response.json();
|
|
const parsed=JSON.parse(data.message?.content||'{}');
|
|
valid=typeof parsed.action==='string'&&typeof parsed.reason==='string'&&typeof parsed.goal==='string';
|
|
action=parsed.action||'';
|
|
reason=parsed.reason||'';
|
|
allowed=scenario.allowed.includes(action);
|
|
}catch(e){error=e instanceof Error?e.message:String(e);}
|
|
rows.push({model,scenario:scenario.id,latencyMs:Math.round(performance.now()-started),validJson:valid,allowedAction:allowed,action,reason:reason.slice(0,160),error});
|
|
}
|
|
}
|
|
console.table(rows.map(({reason,...r})=>r));
|
|
for(const row of rows){
|
|
console.log('\n['+row.model+' / '+row.scenario+'] '+(row.error?'ERROR '+row.error:row.action+' — '+row.reason));
|
|
}
|