加入多模态图片识别功能

This commit is contained in:
2026-09-02 10:52:04 +08:00
parent 10e48fbade
commit f895801ad2
12 changed files with 746 additions and 22 deletions
+35 -8
View File
@@ -2,7 +2,7 @@
// 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。
const fs = require('fs');
const path = require('path');
const { buildMessages, callModel } = require('./llm');
const { buildMessages, callModel, encodeBlocks } = require('./llm');
const { executeTool, resolveInTask } = require('./tools/executor');
const mcp = require('./mcp');
const log = require('./log');
@@ -185,6 +185,12 @@ function modelDisplayName(modelId) {
return row ? row.display_name : modelId;
}
// 会话绑定的模型是否支持 vision(图片回传);决定 read_file 读图行为(见 tools/executor.js)。
function modelVision(session) {
const m = session.model_id && (cfg.models || []).find(x => x.id === session.model_id);
return !!(m && m.vision);
}
function enqueue(sessionId) {
const exists = db.prepare('SELECT id FROM wait_queue WHERE session_id = ?').get(sessionId);
if (!exists) db.prepare('INSERT INTO wait_queue (session_id, queued_at) VALUES (?,?)').run(sessionId, now());
@@ -419,6 +425,7 @@ async function runLoopInner(sessionId, run) {
}
// 执行工具
const vision = modelVision(session);
for (const tc of msgRow.tool_calls) {
if (run.cancelled) break;
const id = tc.id || ('call_' + Math.random().toString(36).slice(2, 10));
@@ -427,16 +434,36 @@ async function runLoopInner(sessionId, run) {
let output;
let isError = false;
try {
output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db },
output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db, vision },
tc.function.name, safeParse(tc.function.arguments));
} catch (err) {
output = 'Error: ' + (err && err.message || String(err));
isError = true;
}
if (run.cancelled && !isError) output = (output ? output + '\n' : '') + '(用户已终止任务)';
log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + String(output || '').slice(0, 500));
bus.emit(sessionId, 'tool_result', { tool_call_id: id, name: tc.function.name, content: output.slice(0, 20000), is_error: isError });
saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: output });
// 工具可返回 {text, image_data_url}(read_file 读图且模型支持 vision):
// 落库时把图片以 image_url 内容块拼进 tool 消息(哨兵 JSON,见 llm.js encodeBlocks),
// buildMessages 发请求前还原为 OpenAI 内容块数组。
let outText;
let imageDataUrl = null;
if (output && typeof output === 'object' && output.image_data_url) {
outText = String(output.text || '');
imageDataUrl = String(output.image_data_url);
} else {
outText = String(output == null ? '' : output);
}
if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)';
log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText.slice(0, 500));
const toolContent = imageDataUrl
? encodeBlocks([
{ type: 'text', text: outText },
{ type: 'image_url', image_url: { url: imageDataUrl } },
])
: outText;
bus.emit(sessionId, 'tool_result', {
tool_call_id: id, name: tc.function.name, content: outText.slice(0, 20000),
is_error: isError, image_url: imageDataUrl || undefined,
});
saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: toolContent });
}
iter++;
}
@@ -483,10 +510,10 @@ function sessionContextContent(session) {
const model = session.model_id
? modelDisplayName(session.model_id)
: '尚未绑定(排队中,绑定后自动开始处理)';
// vision 能力随会话上下文告知 AI:不支持时读图只能拿到元信息,免得白跑一轮 read_file
// vision 能力随会话上下文告知 AI:支持时 read_file 读图会回传图片内容,不支持时只能拿到文件信息
const bound = session.model_id && cfg.models.find(m => m.id === session.model_id);
const visionLine = bound
? (bound.vision ? '支持' : '不支持')
? (bound.vision ? '支持(read_file 读图会返回压缩后的图片内容)' : '不支持(read_file 读图只能拿到文件信息)')
: '未知';
return [
'【会话上下文】由系统自动注入,无需回复本消息',