加入多模态图片识别功能
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
# 冒烟测试用配置:上游指向本地 mock
|
||||
# 两个模型都开 vision:让「读图」端到端测试与绑定哪个模型无关(非 vision 分支由单测覆盖)
|
||||
models:
|
||||
- id: ds1
|
||||
display_name: "Mock Model 1"
|
||||
url: "http://127.0.0.1:9999/v1"
|
||||
vision: true
|
||||
- id: ds2
|
||||
display_name: "Mock Model 2"
|
||||
url: "http://127.0.0.1:9999/v1"
|
||||
vision: true
|
||||
|
||||
workspace_root: "./test/tmp/tasks"
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// 规则:
|
||||
// - 历史里已有 tool 结果 -> 回纯文本收尾
|
||||
// - 最后一条 user 消息含 "写文件" -> 返回 write_file 工具调用
|
||||
// - 含 "读图" -> 返回 read_file(pic.png) 工具调用(验证 vision 图片回传)
|
||||
// - 含 "慢" -> 慢速输出(用于终止测试)
|
||||
// - 其他 -> 纯文本 "已收到: <最后一条user消息>"
|
||||
const http = require('http');
|
||||
@@ -40,6 +41,17 @@ function streamToolCall(res) {
|
||||
res.end();
|
||||
}
|
||||
|
||||
function streamReadFileCall(res) {
|
||||
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
||||
const id = 'call-img-' + Date.now();
|
||||
sse(res, { choices: [{ delta: { role: 'assistant', tool_calls: [{ id, function: { name: 'read_file' } }] } }] });
|
||||
sse(res, { choices: [{ delta: { tool_calls: [{ function: { arguments: '{"path": "pic.png"}' } }] } }] });
|
||||
sse(res, { choices: [{ delta: {}, finish_reason: 'tool_calls' }] });
|
||||
sse(res, { usage: { prompt_tokens: 60, completion_tokens: 10, total_tokens: 70 } });
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
if (!req.url.includes('/chat/completions')) {
|
||||
res.writeHead(404).end('not found');
|
||||
@@ -60,6 +72,8 @@ const server = http.createServer((req, res) => {
|
||||
process.stdout = process.stdout;
|
||||
if (hasToolResult) {
|
||||
streamText(res, '文件已创建并验证完毕。');
|
||||
} else if (userText.includes('读图')) {
|
||||
streamReadFileCall(res);
|
||||
} else if (userText.includes('写文件')) {
|
||||
streamToolCall(res);
|
||||
} else if (userText.includes('慢')) {
|
||||
|
||||
@@ -118,6 +118,26 @@ async function main() {
|
||||
ok(!!taskMd, '开始执行自动追加模板消息');
|
||||
ok(d.session.phase === 'execution', '阶段切换为 execution');
|
||||
|
||||
// ---- 2.5 read_file 读图回传(vision)----
|
||||
console.log('\n[2.5] 读图回传');
|
||||
const sharp = require('sharp');
|
||||
const pngBuf = await sharp({ create: { width: 8, height: 8, channels: 3, background: { r: 200, g: 30, b: 30 } } }).png().toBuffer();
|
||||
const si = await api('POST', '/api/sessions', { title: '冒烟-读图' });
|
||||
await waitSession(si.id, x => x.session.status === 'active', 5000, '绑定');
|
||||
const fdImg = new FormData();
|
||||
fdImg.append('files', new Blob([pngBuf], { type: 'image/png' }), 'pic.png');
|
||||
const upImg = await fetch(BASE + '/api/sessions/' + si.id + '/files', { method: 'POST', body: fdImg });
|
||||
ok(upImg.ok, '上传图片到任务目录');
|
||||
await api('POST', '/api/sessions/' + si.id + '/messages', { content: '读图 pic.png' });
|
||||
d = await waitSession(si.id, x => x.session.status === 'held' && x.messages.some(m => m.role === 'tool'), 15000, '读图回合完成');
|
||||
const imgTool = d.messages.filter(m => m.role === 'tool')
|
||||
.map(m => { try { return JSON.parse(m.content); } catch (_) { return null; } })
|
||||
.find(j => j && Array.isArray(j.__blocks__));
|
||||
ok(!!imgTool && imgTool.__blocks__.some(b =>
|
||||
b.type === 'image_url' && String(b.image_url && b.image_url.url).startsWith('data:image/jpeg;base64,')),
|
||||
'tool 消息含压缩后的 image_url 内容块',
|
||||
imgTool ? JSON.stringify(imgTool.__blocks__.map(b => b.type)) : '无多模态块');
|
||||
|
||||
// ---- 3. 路径穿越防护 ----
|
||||
console.log('\n[3] 路径防护');
|
||||
let blocked = false;
|
||||
@@ -197,6 +217,15 @@ async function main() {
|
||||
ok(fpath.includes('g.txt'), 'find_path 命中', fpath);
|
||||
const esc = await executeTool(ctx, 'read_file', { path: '../outside.txt' }).catch(e => 'ERR:' + e.message);
|
||||
ok(String(esc).includes('越界'), 'read_file 拒绝越界', String(esc));
|
||||
// read_file 读图:非 vision → 文本提示;vision → {text, image_data_url}(依赖 sharp)
|
||||
const unitPng = await require('sharp')({ create: { width: 4, height: 4, channels: 3, background: { r: 10, g: 120, b: 200 } } }).png().toBuffer();
|
||||
fs.writeFileSync(path.join(ctx.taskDir, 'pic.png'), unitPng);
|
||||
const imgNoVision = await executeTool(ctx, 'read_file', { path: 'pic.png' });
|
||||
ok(typeof imgNoVision === 'string' && imgNoVision.includes('不支持视觉识别'), 'read_file 非 vision 图片返回文本提示', String(imgNoVision).slice(0, 80));
|
||||
const ctxVision = Object.assign({}, ctx, { vision: true });
|
||||
const imgVision = await executeTool(ctxVision, 'read_file', { path: 'pic.png' });
|
||||
ok(imgVision && typeof imgVision === 'object' && String(imgVision.image_data_url).startsWith('data:image/jpeg;base64,'),
|
||||
'read_file vision 图片返回压缩 data URL', imgVision && imgVision.text ? String(imgVision.text).slice(0, 90) : String(imgVision).slice(0, 90));
|
||||
|
||||
// ---- 8. 配置读写 API(阶段 2)----
|
||||
console.log('\n[8] 配置读写');
|
||||
|
||||
Reference in New Issue
Block a user