图片read_file时提示以json返回
This commit is contained in:
@@ -224,7 +224,7 @@ platforms:
|
||||
通用模型字段:
|
||||
|
||||
- `enable: false`:模型不进池,不参与分配。
|
||||
- `vision: false`:读图时不回传像素,只给文字占位。
|
||||
- `vision: false`:读图时不回传像素,回执只给结构化说明(`reason: no_vision`)。
|
||||
- `profile: <名>`:引用 `model_profiles`,不配则使用默认全量 prompt 与工具。
|
||||
- `reasoning_ack_field`:需要把思考链回传给上游时填写,例如 DeepSeek 系的 `reasoning_content`。
|
||||
|
||||
|
||||
+5
-1
@@ -139,6 +139,10 @@ POST /api/sessions/:id/messages
|
||||
发请求时 `image_ref` 现签公网短链(`mintUrl`,每次请求重签、不会中途过期)。
|
||||
`tool` 回执里的未消费图片会**挪出回执**,合并成该批回执之后的一条 `user` 消息
|
||||
(`tool` 角色带图是非标准形态,很多上游会吞图);已消费的图降级为文字占位。
|
||||
- `read_file` 读图的回执是**结构化 JSON**(`tools/executor.js` 的 `imageReceipt`:`ok` /
|
||||
`image_attached` / `reason` / `width` / `height` …),不是散文。发送侧三处改写(engine 同回合去重、
|
||||
`buildMessages` 签外链失败与已消费重放)一律走 `patchImageReceipt` **只改字段**;
|
||||
追加句子会让 JSON 不可解析,模型又退回从自然语言猜「像素到底到没到」。
|
||||
- 占位裁剪(`agent.trim_write_rounds` / `trim_result_rounds`):距尾 ≥N 个 assistant 轮的
|
||||
`write_file.content`、`fetch`/`read_file` 回执换成单行占位。**只改发给上游的副本**,DB 与 UI 不动。
|
||||
- `compressMessages` = `buildMessages` + 末尾一条压缩指令 `user`。**铁律**:压缩请求必须与上一次
|
||||
@@ -165,7 +169,7 @@ POST /api/sessions/:id/messages
|
||||
出现即报配置错(不静默忽略)。
|
||||
- `executeTool(baseCtx, name, args)`:先查 `HANDLERS` 表,未命中再交给 MCP 注册表。
|
||||
`ctx = {sessionId, taskDir, run, cfg, db, vision}`;工具返回字符串,`read_file` 读图时返回
|
||||
`{text, image_rel}`(由 engine 转成 `image_ref` 块)。
|
||||
`{text, image_rel}`(`text` 是结构化 JSON 回执,由 engine 转成 `image_ref` 块)。
|
||||
- **路径沙箱**:所有路径经 `resolveInTask` 解析到 `taskDir` 内;`/tmp` 会被映射到
|
||||
`<taskDir>/.tmp`(与 executor 容器里 `/tmp` 的 bind 对齐)。符号链接越界一律拒绝。
|
||||
- **跨会话只读**:`@<session-id>/<rel>` 只允许 `list_directory`、`read_file`、`copy_path` 的
|
||||
|
||||
+28
-10
@@ -2,7 +2,7 @@
|
||||
// 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { buildMessages, callModel, encodeBlocks, decodeBlocks, compressMessages, titleMessages, setProfilePrompts } = require('./llm');
|
||||
const { buildMessages, callModel, encodeBlocks, decodeBlocks, compressMessages, titleMessages, setProfilePrompts, patchImageReceipt } = require('./llm');
|
||||
const { executeTool, resolveInTask } = require('./tools/executor');
|
||||
const publicFiles = require('./public-files');
|
||||
const mcp = require('./mcp');
|
||||
@@ -771,31 +771,49 @@ async function runLoopInner(sessionId, run) {
|
||||
isError = true;
|
||||
}
|
||||
}
|
||||
// 工具可返回 {text, image_rel}(read_file 读图且模型支持 vision,压缩产物在 .imgcache/):
|
||||
// 落库时以 image_ref 内容块拼进 tool 消息(哨兵 JSON,见 llm.js encodeBlocks);
|
||||
// buildMessages 发请求前把未消费的图现签成公网外链 image_url 块,并入该批回执之后的
|
||||
// 一条 user 消息发送(tool 角色带图非标准形态,上游支持度差);前端工具卡走带 cookie 的下载路由渲染。
|
||||
// 工具可返回 {text, image_rel}(read_file 读图且模型支持 vision,压缩产物在 .imgcache/,
|
||||
// text 是 imageReceipt 生成的结构化 JSON 回执):落库时以 image_ref 内容块拼进 tool 消息
|
||||
// (哨兵 JSON,见 llm.js encodeBlocks);buildMessages 发请求前把未消费的图现签成公网
|
||||
// 外链 image_url 块,并入该批回执之后的一条 user 消息发送(tool 角色带图非标准形态,
|
||||
// 上游支持度差);前端工具卡走带 cookie 的下载路由渲染。
|
||||
let outText;
|
||||
let imageRel = null;
|
||||
if (output && typeof output === 'object' && output.image_rel) {
|
||||
outText = String(output.text || '');
|
||||
imageRel = String(output.image_rel);
|
||||
if (output && typeof output === 'object' && typeof output.text === 'string') {
|
||||
outText = output.text;
|
||||
if (output.image_rel) imageRel = String(output.image_rel);
|
||||
} else {
|
||||
outText = String(output == null ? '' : output);
|
||||
}
|
||||
if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)';
|
||||
// 死循环护栏:同回合内同一张图只投递一次——上游能认多图(探针实测同消息双外链全对),
|
||||
// 不再按批次限流;但模型若因惯性叙事反复重读同一文件,每轮都签新图就是纯烧 token,
|
||||
// 故已投递过的图再读只回文字提示(run 随回合销毁,新回合自动重置,追问重看不受影响)。
|
||||
// 故已投递过的图再读只回执「未附加 + 原因」。改写回执的字段而不是追加句子:
|
||||
// 追加会把 JSON 弄成不可解析的尾巴,模型又只能靠语义猜。run 随回合销毁,
|
||||
// 新回合自动重置,追问重看不受影响。
|
||||
if (imageRel) {
|
||||
if (!run.imgDelivered) run.imgDelivered = new Set();
|
||||
if (run.imgDelivered.has(imageRel)) {
|
||||
outText += '\n[该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件]';
|
||||
const patched = patchImageReceipt(outText, {
|
||||
image_attached: false, reason: 'already_delivered', retryable: false,
|
||||
hint: '该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件',
|
||||
});
|
||||
outText = patched !== null
|
||||
? patched
|
||||
: outText + '\n[该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件]';
|
||||
imageRel = null;
|
||||
} else {
|
||||
run.imgDelivered.add(imageRel);
|
||||
}
|
||||
}
|
||||
// 回执自报的「环境故障」也算失败(UI 打 ✕、库里 is_error=true),预期拒绝不算:
|
||||
// no_vision / too_large / past_session / already_delivered / link_unavailable 都是正常答复。
|
||||
// 限定工具名再解析,避免把 db 的大块查询结果白解一遍。
|
||||
if (!isError && tc.function.name === 'read_file' && typeof outText === 'string' && outText.charAt(0) === '{') {
|
||||
try {
|
||||
const o = JSON.parse(outText);
|
||||
if (o && o.kind === 'image' && o.ok === false && (o.reason === 'decode_failed' || o.reason === 'sharp_missing')) isError = true;
|
||||
} catch (_) { /* 非结构化回执(旧格式)不处理 */ }
|
||||
}
|
||||
log.debug('tool', sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText);
|
||||
const toolContent = imageRel
|
||||
? encodeBlocks([
|
||||
|
||||
+49
-6
@@ -126,13 +126,38 @@ function cachedDecodeBlocks(session, row) {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
// read_file 图片回执是单个 JSON 对象(字段契约见 tools/executor.js 的 imageReceipt)。
|
||||
// 发送侧要改写回执时只动字段,绝不追加句子:拼在 JSON 后面会让整条回执不可解析,
|
||||
// 模型又退回「从自然语言里猜像素到底到没到」。非 JSON / 非图片回执(旧库散文、文本读取正文)
|
||||
// 返回 null,调用方自行回退。
|
||||
function patchImageReceipt(text, patch) {
|
||||
if (typeof text !== 'string' || text.charAt(0) !== '{') return null;
|
||||
let o;
|
||||
try { o = JSON.parse(text); } catch (_) { return null; }
|
||||
if (!o || o.kind !== 'image') return null;
|
||||
return JSON.stringify(Object.assign(o, patch));
|
||||
}
|
||||
// 图片编号标签:从回执字段生成「read_file <path> 1080×1920」,只用于把图与回执对上号。
|
||||
// 旧库散文回执退回首行(历史行照旧可读)。
|
||||
function imageLabel(text) {
|
||||
if (typeof text === 'string' && text.charAt(0) === '{') {
|
||||
try {
|
||||
const o = JSON.parse(text);
|
||||
if (o && o.kind === 'image') return 'read_file ' + (o.path || '') + (o.width && o.height ? ' ' + o.width + '×' + o.height : '');
|
||||
} catch (_) { /* 落到下面按散文取首行 */ }
|
||||
}
|
||||
return (text || 'read_file 图片回传').split('\n')[0];
|
||||
}
|
||||
|
||||
// 将 DB 消息行转为 OpenAI messages 数组(不含 system,由调用方拼接)。
|
||||
// 图片两种形态:旧库/用户粘贴附件的 image_url 块(data: URL)原样透传;
|
||||
// 新版 read_file 读图存的是 image_ref 块({rel},指向任务目录 .imgcache/ 产物)。
|
||||
// tool 回执里的未消费图不放在回执内发送——tool 角色带 image_url 非标准形态,
|
||||
// 大量 OpenAI 兼容上游会整批忽略或只认批次最后一张,模型看不到图却见回执写着
|
||||
// 「已回传」,会反复 read_file 补看(死循环烧 token,见多模态回退复盘);
|
||||
// 故回执降级为纯文本,图块现签成 image_url(mintUrl 公网链回调)后统一并入
|
||||
// 故回执里只留文字(read_file 读图时是一个结构化 JSON 对象,字段见
|
||||
// tools/executor.js 的 imageReceipt;发送侧只改字段、不追加句子),图块现签成
|
||||
// image_url(mintUrl 公网链回调)后统一并入
|
||||
// 该批之后的一条 user 消息(user 角色各家通吃):逐图说明文字全部置顶、图块按回执顺序紧随
|
||||
//(对齐官方示例排布;探针实测 DS 对交替排布也兼容,取低风险形态),
|
||||
// 并附「先逐张简述再继续任务」指令——兼作送达自检,破弱模型的“没收到”惯性叙事;
|
||||
@@ -267,13 +292,21 @@ function buildMessages(session, language, mintUrl, profile, withProfile, trimOn)
|
||||
let content = row.content || '';
|
||||
if (blocks) {
|
||||
if (i < lastAssistant) {
|
||||
content = blocks.filter(b => b && b.type === 'text').map(b => b.text).join('\n') +
|
||||
'\n[图片数据已省略:本图已随上一条回执发送过,不再重复携带;若你当时未能识别其内容,请直接告知用户,勿反复重读同一文件]';
|
||||
content = blocks.filter(b => b && b.type === 'text').map(b => b.text).join('\n');
|
||||
// 已消费的图重放时不再携带像素:把回执的 image_attached 降为 false 并给出原因,
|
||||
// 模型读字段就知道「不是又丢了」,不会因叙事惯性重读。旧库散文回执保持原有尾巴。
|
||||
const consumed = patchImageReceipt(content, {
|
||||
image_attached: false, reason: 'omitted_already_sent', retryable: false,
|
||||
hint: '该图已在此前的轮次投递过,重放时不再重复携带像素;若你当时未能识别,请直接告知用户,勿反复重读同一文件',
|
||||
});
|
||||
content = consumed !== null ? consumed
|
||||
: content + '\n[图片数据已省略:本图已随上一条回执发送过,不再重复携带;若你当时未能识别其内容,请直接告知用户,勿反复重读同一文件]';
|
||||
} else if (row.role === 'tool') {
|
||||
// 未消费的图:回执只留文字,图块挪进批次后的 user 合并消息(见 pendingImages/Labels)
|
||||
const texts = blocks.filter(b => b && b.type === 'text').map(b => b.text);
|
||||
content = texts.join('\n');
|
||||
const label = (texts[0] || 'read_file 图片回传').split('\n')[0];
|
||||
const label = imageLabel(texts[0]);
|
||||
let mintFailed = false;
|
||||
for (const b of blocks) {
|
||||
if (!b || (b.type !== 'image_ref' && b.type !== 'image_url')) continue;
|
||||
const seq = ++imgSeq;
|
||||
@@ -287,9 +320,19 @@ function buildMessages(session, language, mintUrl, profile, withProfile, trimOn)
|
||||
pendingLabels.push('[图片 ' + seq + '] ' + label);
|
||||
pendingImages.push({ type: 'image_url', image_url: { url } });
|
||||
} else {
|
||||
pendingLabels.push('[图片 ' + seq + '] ' + label + '(外链不可用,未取得像素:请直接告知用户,勿反复重读同一文件)');
|
||||
mintFailed = true;
|
||||
pendingLabels.push('[图片 ' + seq + '] ' + label + '(未送达:link_unavailable)');
|
||||
}
|
||||
}
|
||||
// 签不到外链 = 本次请求像素确实到不了:当场把回执字段降级,让模型拿到的是
|
||||
// 「ok 但 image_attached=false + reason」,而不是一句需要它自己解读的说明。
|
||||
if (mintFailed) {
|
||||
const degraded = patchImageReceipt(content, {
|
||||
image_attached: false, reason: 'link_unavailable', retryable: false,
|
||||
hint: '服务端签不出图片外链,本张图的像素未能随本次请求送达;请直接告知用户,勿反复重读同一文件',
|
||||
});
|
||||
if (degraded !== null) content = degraded;
|
||||
}
|
||||
} else {
|
||||
// user 行(粘贴附件/旧形态):image_url 块(data: URL)原样;image_ref → 现签公网外链
|
||||
//(每次请求重签,不过期)
|
||||
@@ -646,4 +689,4 @@ async function callModel(cfg, modelId, messages, tools, options) {
|
||||
return { content, reasoning, tool_calls: Array.from(toolCalls.values()), finish_reason: finishReason, usage, aborted, sseFrames };
|
||||
}
|
||||
|
||||
module.exports = { callModel, buildMessages, buildSystemPrompt, systemPromptText, setSkillListText, setPayloadTrim, setProfilePrompts, encodeBlocks, decodeBlocks, compressMessages, titleMessages };
|
||||
module.exports = { callModel, buildMessages, buildSystemPrompt, systemPromptText, setSkillListText, setPayloadTrim, setProfilePrompts, encodeBlocks, decodeBlocks, compressMessages, titleMessages, patchImageReceipt };
|
||||
|
||||
@@ -95,8 +95,10 @@ When the user asks for something that seems out of scope:
|
||||
carry state across calls.
|
||||
- Python 3.11 is available; the system Python and its packages are read-only.
|
||||
To add packages, create a virtualenv inside the working directory
|
||||
(`python3 -m venv .venv`, then `.venv/bin/pip` and `.venv/bin/python`). Keep
|
||||
it in the working directory, not `/tmp`.
|
||||
(`python3 -m venv --system-site-packages .venv`, then `.venv/bin/pip` and `.venv/bin/python`).
|
||||
Keep it in the working directory, not `/tmp`.
|
||||
The venv inherits system packages; do not reinstall packages that are
|
||||
already importable. Only install what is missing.
|
||||
- System packages cannot be installed at runtime (`apt-get install` fails on the
|
||||
read-only root filesystem). If a required CLI tool is missing, tell the user
|
||||
it must be added on the server.
|
||||
|
||||
@@ -219,7 +219,7 @@
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nCross-session peek: `path` accepts '@<session-id>/<rel>' to read a text file from your OWN past session, read-only (find the id with recall op=search, browse with list_directory). It is a look, not an import: nothing lands in your session directory, so `terminal`, grep and `edit_file` cannot see it, and past images are refused. When you need to work with a file or view a past image, `copy_path` it first (its `source_path` takes the same '@' form). '@' is never accepted by `write_file` or `edit_file`.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF) are returned as visual content you can analyze directly, but only if the current model supports vision; otherwise a text placeholder is returned — that placeholder is not an error. The tool result itself carries a text notice only; the actual image pixels arrive in a separate user message right after the batch of tool results: that message lists per-image labels matching each image to its read_file receipt, then the image blocks in receipt order, and asks you to briefly describe each image before continuing. Never re-read an image already delivered in the current turn; if pixels never arrive, tell the user instead of retrying. Very large images (roughly over 50 MB) are skipped, and images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before being returned.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).",
|
||||
"description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nCross-session peek: `path` accepts '@<session-id>/<rel>' to read a text file from your OWN past session, read-only (find the id with recall op=search, browse with list_directory). It is a look, not an import: nothing lands in your session directory, so `terminal`, grep and `edit_file` cannot see it, and past images are refused. When you need to work with a file or view a past image, `copy_path` it first (its `source_path` takes the same '@' form). '@' is never accepted by `write_file` or `edit_file`.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF): the receipt is a single JSON object, not prose — read its fields instead of interpreting the text. Success shape: {\"ok\":true,\"tool\":\"read_file\",\"kind\":\"image\",\"path\":\"...\",\"image_attached\":true,\"width\":1080,\"height\":1920,\"format\":\"jpeg\",\"bytes_in\":3550000,\"bytes_out\":210000,\"resized\":true,\"source_width\":4000,\"source_height\":7111}. `width`/`height` are the delivered pixel size; `source_width`/`source_height` appear only when the image was downscaled. `image_attached:true` means the server produced the image and it will arrive as an image block in the separate user message that immediately follows the batch of tool results; that message labels each image `[图片 N] read_file <path> <w>×<h>` and asks you to describe it before continuing. The tool result itself never carries pixels. Refusal or failure shape: {\"ok\":false,\"kind\":\"image\",\"path\":\"...\",\"image_attached\":false,\"reason\":\"...\",\"retryable\":false,\"hint\":\"...\"} where `reason` is one of no_vision (this model cannot see images), too_large (roughly over 50 MB), past_session (import it with copy_path first), decode_failed, sharp_missing, already_delivered (the same image was already sent earlier in this turn), link_unavailable (no public URL could be minted, so the pixels did not arrive for that request), omitted_already_sent (an older receipt whose image was already delivered; replay omits the pixels). `ok:true` with `image_attached:false` is still a successful call, e.g. a deduplicated re-read. When `retryable` is false do not retry — tell the user instead. Images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before delivery.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
|
||||
@@ -129,6 +129,9 @@ const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.
|
||||
// read_file 读图(仅当会话绑定的模型 vision: true):sharp 压缩后落盘到任务目录 .imgcache/,
|
||||
// 返回 {text, image_rel};engine 拼成 image_ref 内容块落库,llm.buildMessages 发请求前
|
||||
// 现签成公网外链(GET /p/<id>)进 image_url 块,base64 不再进 DB/请求体。
|
||||
// text 一律是 imageReceipt 生成的 JSON 对象(字段契约见下),不是散文:
|
||||
// 这份回执会被三层改写(executor 产出 / engine 同回合去重 / llm 签外链失败降级),
|
||||
// 散文回执让模型只能靠语义猜「到底有没有图」,历史事故是反复 read_file 补看烧 token。
|
||||
const IMG_MAX_INPUT_BYTES = 50 * 1024 * 1024; // 与上传上限一致
|
||||
const IMG_MAX_SIDE = 2000; // 长边压缩阈值(px)
|
||||
const IMG_JPEG_QUALITY = 80;
|
||||
@@ -178,14 +181,51 @@ function outlineWrapper(outlineText) {
|
||||
'For example, to read a function shown as [L100-150], use start_line: 100 and end_line: 150.';
|
||||
}
|
||||
|
||||
// read_file 图片回执的统一信封。字段是模型唯一可判定的事实,回执不再写中文句子:
|
||||
// ok 调用本身是否成功(ok=true + image_attached=false 是合法形态:同回合重复读被去重)
|
||||
// path 模型请求的路径(@ 历史形原样保留);投递产物路径只在服务端内部用,不回执
|
||||
// image_attached 服务端是否产出了图块、会随本批回执之后的那条 user 消息投递(≠上游已收到)
|
||||
// reason 仅 image_attached=false 时有值,稳定枚举(no_vision / too_large / past_session /
|
||||
// decode_failed / sharp_missing / already_delivered / link_unavailable /
|
||||
// omitted_already_sent)——模型按它分支,不要读 hint 猜
|
||||
// retryable 重试同一调用是否有意义(图片这条路上恒 false:同样的字节只会得到同样的结果)
|
||||
// width/height 实际投递的像素尺寸;source_width/height + resized 只在确实缩放过时出现
|
||||
// hint 一句人话补充,仅供解释,不承载判定
|
||||
function imageReceipt(o) {
|
||||
const out = { ok: o.ok === true, tool: 'read_file', kind: 'image', path: o.path, image_attached: o.image_attached === true };
|
||||
for (const k of ['reason', 'retryable', 'width', 'height', 'format', 'bytes_in', 'bytes_out',
|
||||
'resized', 'source_width', 'source_height', 'hint']) {
|
||||
if (o[k] !== undefined) out[k] = o[k];
|
||||
}
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
// 用 sharp 压缩(按 EXIF 摆正、长边 ≤2000px、JPEG 质量 80)并落盘 .imgcache/。
|
||||
// 成功返回 {text, image_rel};产物文件名带源文件指纹(mtime+size),同图重复读复用同一份不增殖。
|
||||
// 成功返回 {text, image_rel},失败返回 {text}(text 都是 imageReceipt 的 JSON)。
|
||||
// 产物文件名带源文件指纹(mtime+size),同图重复读复用同一份不增殖。
|
||||
// 无收益守卫(与浏览器端压缩同款):重编码 JPEG 反而 ≥ 原图时不落盘,image_rel 直指原文件。
|
||||
// sharp 缺失或解码失败时退回纯文本提示,不让整个工具回合报错。
|
||||
// sharp 缺失或解码失败时只回执原因,不让整个工具回合报错。
|
||||
async function readImageAsBlocks(ctx, abs, rel, size) {
|
||||
let sharp;
|
||||
try { sharp = require('sharp'); }
|
||||
catch (_) { return '[图片文件] ' + rel + '(' + size + ' 字节)。服务端未安装 sharp,无法压缩回传图片内容。'; }
|
||||
catch (_) {
|
||||
return {
|
||||
text: imageReceipt({
|
||||
ok: false, path: rel, image_attached: false, reason: 'sharp_missing', retryable: false, bytes_in: size,
|
||||
hint: '服务端未安装 sharp,无法压缩回传图片内容',
|
||||
}),
|
||||
};
|
||||
}
|
||||
// 源尺寸只读文件头(metadata 不解码像素);EXIF orientation 5-8 表示显示时旋转 90°,
|
||||
// 宽高要交换才与模型最终看到的画面一致(sharp 的 rotate() 会真的转)。
|
||||
let src = null;
|
||||
try {
|
||||
const m = await sharp(abs).metadata();
|
||||
if (m && m.width && m.height) {
|
||||
const swap = Number(m.orientation) >= 5;
|
||||
src = { w: swap ? m.height : m.width, h: swap ? m.width : m.height, format: m.format || null };
|
||||
}
|
||||
} catch (_) { src = null; }
|
||||
try {
|
||||
const r = await sharp(abs)
|
||||
.rotate()
|
||||
@@ -195,7 +235,13 @@ async function readImageAsBlocks(ctx, abs, rel, size) {
|
||||
if (r.data.length >= size) {
|
||||
// 低内容熵图(截图/线稿/纯色)JPEG 重编码可能比原图还大:不产出缓存,外链直接指向原文件
|
||||
return {
|
||||
text: '[图片文件] ' + rel + '(' + size + ' 字节,压缩无收益,按原图回传)',
|
||||
text: imageReceipt({
|
||||
ok: true, path: rel, image_attached: true,
|
||||
width: src ? src.w : undefined, height: src ? src.h : undefined,
|
||||
format: src && src.format ? src.format : undefined,
|
||||
bytes_in: size, bytes_out: size, resized: false,
|
||||
hint: '压缩无收益,按原图回传',
|
||||
}),
|
||||
image_rel: rel,
|
||||
};
|
||||
}
|
||||
@@ -208,13 +254,21 @@ async function readImageAsBlocks(ctx, abs, rel, size) {
|
||||
await fsp.mkdir(path.dirname(outAbs), { recursive: true });
|
||||
await fsp.writeFile(outAbs, r.data);
|
||||
}
|
||||
return {
|
||||
text: '[图片文件] ' + rel + '(原 ' + size + ' 字节,已压缩为 ' + r.info.width + '×' + r.info.height +
|
||||
' JPEG ' + r.data.length + ' 字节)',
|
||||
image_rel: cacheRel,
|
||||
const receipt = {
|
||||
ok: true, path: rel, image_attached: true,
|
||||
width: r.info.width, height: r.info.height, format: 'jpeg',
|
||||
bytes_in: size, bytes_out: r.data.length,
|
||||
resized: src ? (r.info.width !== src.w || r.info.height !== src.h) : undefined,
|
||||
};
|
||||
if (src) { receipt.source_width = src.w; receipt.source_height = src.h; }
|
||||
return { text: imageReceipt(receipt), image_rel: cacheRel };
|
||||
} catch (err) {
|
||||
return '[图片文件] ' + rel + '(' + size + ' 字节),图片解码/压缩失败:' + (err && err.message || err);
|
||||
return {
|
||||
text: imageReceipt({
|
||||
ok: false, path: rel, image_attached: false, reason: 'decode_failed', retryable: false, bytes_in: size,
|
||||
hint: '图片解码/压缩失败:' + (err && err.message || err),
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,15 +298,29 @@ async function toolReadFile(ctx, args) {
|
||||
// 不做「偷偷复制」:图片要出现在模型视野里,字节必须落到本会话才能签外链(engine 的
|
||||
// mintImageUrl 与前端下载路由都以当前 taskDir 为根),那本质已是一次导入——与其藏进
|
||||
// .imgcache 不进文件表,不如让模型明确知道历史图片必须先 copy_path。
|
||||
return '[图片文件] ' + disp + '(' + st.size + ' 字节)历史会话图片不支持直接读取。' +
|
||||
'图片需要先 copy_path(source_path 用这个 @ 路径)导入本会话,再 read_file——导入后图片、terminal、mint 全量生效。';
|
||||
return {
|
||||
text: imageReceipt({
|
||||
ok: false, path: disp, image_attached: false, reason: 'past_session', retryable: false, bytes_in: st.size,
|
||||
hint: '历史会话图片不支持直接读取:先 copy_path(source_path 用这个 @ 路径)导入本会话,再 read_file——导入后图片、terminal、mint 全量生效',
|
||||
}),
|
||||
};
|
||||
}
|
||||
const rel = relOf(ctx.taskDir, abs);
|
||||
if (!ctx.vision) {
|
||||
return '[图片文件] ' + rel + '(' + st.size + ' 字节)。图片文件,当前模型不支持视觉识别,无法回传图片内容。';
|
||||
return {
|
||||
text: imageReceipt({
|
||||
ok: false, path: rel, image_attached: false, reason: 'no_vision', retryable: false, bytes_in: st.size,
|
||||
hint: '当前模型不支持视觉识别,无法回传图片像素;请基于文件名/大小作答,或直接告知用户',
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (st.size > IMG_MAX_INPUT_BYTES) {
|
||||
return '[图片文件] ' + rel + '(' + st.size + ' 字节)过大(上限 ' + IMG_MAX_INPUT_BYTES + ' 字节),已跳过回传。';
|
||||
return {
|
||||
text: imageReceipt({
|
||||
ok: false, path: rel, image_attached: false, reason: 'too_large', retryable: false, bytes_in: st.size,
|
||||
hint: '超过回传上限 ' + IMG_MAX_INPUT_BYTES + ' 字节;可先用 terminal 缩小(如 ImageMagick)再读',
|
||||
}),
|
||||
};
|
||||
}
|
||||
return await readImageAsBlocks(ctx, abs, rel, st.size);
|
||||
}
|
||||
@@ -1660,7 +1728,8 @@ const HANDLERS = {
|
||||
};
|
||||
|
||||
// ctx: {sessionId, taskDir, run, cfg, db, vision}
|
||||
// 返回值一般为字符串;read_file 读图且 ctx.vision 为 true 时返回 {text, image_rel}。
|
||||
// 返回值一般为字符串;read_file 读图时返回 {text, image_rel},text 是结构化 JSON 回执
|
||||
// (见 imageReceipt),image_rel 缺省表示本次不投递像素。
|
||||
// 内置工具优先;未命中时交给 MCP 注册表(启动时发现的服务工具)。
|
||||
async function executeTool(baseCtx, name, args) {
|
||||
const handler = HANDLERS[name];
|
||||
|
||||
@@ -65,3 +65,13 @@ system prompt 不重复 schema 已表达的内容。
|
||||
### 八、上下文成本
|
||||
|
||||
tools 数组每轮请求完整重发。新工具进来时,先问"能不能装进已有工具",再问"要不要新建"。
|
||||
|
||||
### 九、回执形态
|
||||
|
||||
- **内容型**工具(`read_file` 的文本、`fetch` 的正文、`db` 的查询结果)回执就是内容本身,不要套 JSON 信封。
|
||||
- **结果型**动作(读图、裁剪、转换)回执用**单个 JSON 对象**,把判定写进字段,不写自然语言:模型按字段分支,
|
||||
不要再让它从句子里猜。字段名跨工具保持一致:`ok`(调用是否成功)/ `reason`(稳定枚举)/ `retryable` /
|
||||
`width` `height` 之类的事实字段 / `hint`(仅解释,不承载判定)。
|
||||
- 回执可能被多层改写(发送前裁剪、同回合去重、投递失败降级):改写**只动字段**,不得在 JSON 后追加句子——
|
||||
追加会把回执变成不可解析的尾巴,等于把判定权又还给语义猜测。
|
||||
- `ok:true` 不等于「结果已送达」:另有字段(如 `image_attached`)描述投递状态,两者不要混成一个布尔。
|
||||
|
||||
Reference in New Issue
Block a user