diff --git a/README.md b/README.md index 3b4a645..2d5eeb3 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ platforms: 通用模型字段: - `enable: false`:模型不进池,不参与分配。 -- `vision: false`:读图时不回传像素,只给文字占位。 +- `vision: false`:读图时不回传像素,回执只给结构化说明(`reason: no_vision`)。 - `profile: <名>`:引用 `model_profiles`,不配则使用默认全量 prompt 与工具。 - `reasoning_ack_field`:需要把思考链回传给上游时填写,例如 DeepSeek 系的 `reasoning_content`。 diff --git a/app/AGENTS.md b/app/AGENTS.md index e9ac47a..83dd30f 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -139,6 +139,10 @@ POST /api/sessions/:id/messages 发请求时 `image_ref` 现签公网短链(`mintUrl`,每次请求重签、不会中途过期)。 `tool` 回执里的未消费图片会**挪出回执**,合并成该批回执之后的一条 `user` 消息 (`tool` 角色带图是非标准形态,很多上游会吞图);已消费的图降级为文字占位。 +- `read_file` 读图的回执是**结构化 JSON**(`tools/executor.js` 的 `imageReceipt`:`ok` / + `image_attached` / `reason` / `width` / `height` …),不是散文。发送侧三处改写(engine 同回合去重、 + `buildMessages` 签外链失败与已消费重放)一律走 `patchImageReceipt` **只改字段**; + 追加句子会让 JSON 不可解析,模型又退回从自然语言猜「像素到底到没到」。 - 占位裁剪(`agent.trim_write_rounds` / `trim_result_rounds`):距尾 ≥N 个 assistant 轮的 `write_file.content`、`fetch`/`read_file` 回执换成单行占位。**只改发给上游的副本**,DB 与 UI 不动。 - `compressMessages` = `buildMessages` + 末尾一条压缩指令 `user`。**铁律**:压缩请求必须与上一次 @@ -165,7 +169,7 @@ POST /api/sessions/:id/messages 出现即报配置错(不静默忽略)。 - `executeTool(baseCtx, name, args)`:先查 `HANDLERS` 表,未命中再交给 MCP 注册表。 `ctx = {sessionId, taskDir, run, cfg, db, vision}`;工具返回字符串,`read_file` 读图时返回 - `{text, image_rel}`(由 engine 转成 `image_ref` 块)。 + `{text, image_rel}`(`text` 是结构化 JSON 回执,由 engine 转成 `image_ref` 块)。 - **路径沙箱**:所有路径经 `resolveInTask` 解析到 `taskDir` 内;`/tmp` 会被映射到 `/.tmp`(与 executor 容器里 `/tmp` 的 bind 对齐)。符号链接越界一律拒绝。 - **跨会话只读**:`@/` 只允许 `list_directory`、`read_file`、`copy_path` 的 diff --git a/app/server/engine.js b/app/server/engine.js index 16f2784..13fea73 100644 --- a/app/server/engine.js +++ b/app/server/engine.js @@ -2,7 +2,7 @@ // 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。 const fs = require('fs'); const path = require('path'); -const { buildMessages, callModel, encodeBlocks, decodeBlocks, compressMessages, titleMessages, setProfilePrompts } = require('./llm'); +const { buildMessages, callModel, encodeBlocks, decodeBlocks, compressMessages, titleMessages, setProfilePrompts, patchImageReceipt } = require('./llm'); const { executeTool, resolveInTask } = require('./tools/executor'); const publicFiles = require('./public-files'); const mcp = require('./mcp'); @@ -771,31 +771,49 @@ async function runLoopInner(sessionId, run) { isError = true; } } - // 工具可返回 {text, image_rel}(read_file 读图且模型支持 vision,压缩产物在 .imgcache/): - // 落库时以 image_ref 内容块拼进 tool 消息(哨兵 JSON,见 llm.js encodeBlocks); - // buildMessages 发请求前把未消费的图现签成公网外链 image_url 块,并入该批回执之后的 - // 一条 user 消息发送(tool 角色带图非标准形态,上游支持度差);前端工具卡走带 cookie 的下载路由渲染。 + // 工具可返回 {text, image_rel}(read_file 读图且模型支持 vision,压缩产物在 .imgcache/, + // text 是 imageReceipt 生成的结构化 JSON 回执):落库时以 image_ref 内容块拼进 tool 消息 + // (哨兵 JSON,见 llm.js encodeBlocks);buildMessages 发请求前把未消费的图现签成公网 + // 外链 image_url 块,并入该批回执之后的一条 user 消息发送(tool 角色带图非标准形态, + // 上游支持度差);前端工具卡走带 cookie 的下载路由渲染。 let outText; let imageRel = null; - if (output && typeof output === 'object' && output.image_rel) { - outText = String(output.text || ''); - imageRel = String(output.image_rel); + if (output && typeof output === 'object' && typeof output.text === 'string') { + outText = output.text; + if (output.image_rel) imageRel = String(output.image_rel); } else { outText = String(output == null ? '' : output); } if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)'; // 死循环护栏:同回合内同一张图只投递一次——上游能认多图(探针实测同消息双外链全对), // 不再按批次限流;但模型若因惯性叙事反复重读同一文件,每轮都签新图就是纯烧 token, - // 故已投递过的图再读只回文字提示(run 随回合销毁,新回合自动重置,追问重看不受影响)。 + // 故已投递过的图再读只回执「未附加 + 原因」。改写回执的字段而不是追加句子: + // 追加会把 JSON 弄成不可解析的尾巴,模型又只能靠语义猜。run 随回合销毁, + // 新回合自动重置,追问重看不受影响。 if (imageRel) { if (!run.imgDelivered) run.imgDelivered = new Set(); if (run.imgDelivered.has(imageRel)) { - outText += '\n[该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件]'; + const patched = patchImageReceipt(outText, { + image_attached: false, reason: 'already_delivered', retryable: false, + hint: '该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件', + }); + outText = patched !== null + ? patched + : outText + '\n[该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件]'; imageRel = null; } else { run.imgDelivered.add(imageRel); } } + // 回执自报的「环境故障」也算失败(UI 打 ✕、库里 is_error=true),预期拒绝不算: + // no_vision / too_large / past_session / already_delivered / link_unavailable 都是正常答复。 + // 限定工具名再解析,避免把 db 的大块查询结果白解一遍。 + if (!isError && tc.function.name === 'read_file' && typeof outText === 'string' && outText.charAt(0) === '{') { + try { + const o = JSON.parse(outText); + if (o && o.kind === 'image' && o.ok === false && (o.reason === 'decode_failed' || o.reason === 'sharp_missing')) isError = true; + } catch (_) { /* 非结构化回执(旧格式)不处理 */ } + } log.debug('tool', sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText); const toolContent = imageRel ? encodeBlocks([ diff --git a/app/server/llm.js b/app/server/llm.js index df0674e..5327592 100644 --- a/app/server/llm.js +++ b/app/server/llm.js @@ -126,13 +126,38 @@ function cachedDecodeBlocks(session, row) { return blocks; } +// read_file 图片回执是单个 JSON 对象(字段契约见 tools/executor.js 的 imageReceipt)。 +// 发送侧要改写回执时只动字段,绝不追加句子:拼在 JSON 后面会让整条回执不可解析, +// 模型又退回「从自然语言里猜像素到底到没到」。非 JSON / 非图片回执(旧库散文、文本读取正文) +// 返回 null,调用方自行回退。 +function patchImageReceipt(text, patch) { + if (typeof text !== 'string' || text.charAt(0) !== '{') return null; + let o; + try { o = JSON.parse(text); } catch (_) { return null; } + if (!o || o.kind !== 'image') return null; + return JSON.stringify(Object.assign(o, patch)); +} +// 图片编号标签:从回执字段生成「read_file 1080×1920」,只用于把图与回执对上号。 +// 旧库散文回执退回首行(历史行照旧可读)。 +function imageLabel(text) { + if (typeof text === 'string' && text.charAt(0) === '{') { + try { + const o = JSON.parse(text); + if (o && o.kind === 'image') return 'read_file ' + (o.path || '') + (o.width && o.height ? ' ' + o.width + '×' + o.height : ''); + } catch (_) { /* 落到下面按散文取首行 */ } + } + return (text || 'read_file 图片回传').split('\n')[0]; +} + // 将 DB 消息行转为 OpenAI messages 数组(不含 system,由调用方拼接)。 // 图片两种形态:旧库/用户粘贴附件的 image_url 块(data: URL)原样透传; // 新版 read_file 读图存的是 image_ref 块({rel},指向任务目录 .imgcache/ 产物)。 // tool 回执里的未消费图不放在回执内发送——tool 角色带 image_url 非标准形态, // 大量 OpenAI 兼容上游会整批忽略或只认批次最后一张,模型看不到图却见回执写着 // 「已回传」,会反复 read_file 补看(死循环烧 token,见多模态回退复盘); -// 故回执降级为纯文本,图块现签成 image_url(mintUrl 公网链回调)后统一并入 +// 故回执里只留文字(read_file 读图时是一个结构化 JSON 对象,字段见 +// tools/executor.js 的 imageReceipt;发送侧只改字段、不追加句子),图块现签成 +// image_url(mintUrl 公网链回调)后统一并入 // 该批之后的一条 user 消息(user 角色各家通吃):逐图说明文字全部置顶、图块按回执顺序紧随 //(对齐官方示例排布;探针实测 DS 对交替排布也兼容,取低风险形态), // 并附「先逐张简述再继续任务」指令——兼作送达自检,破弱模型的“没收到”惯性叙事; @@ -267,13 +292,21 @@ function buildMessages(session, language, mintUrl, profile, withProfile, trimOn) let content = row.content || ''; if (blocks) { if (i < lastAssistant) { - content = blocks.filter(b => b && b.type === 'text').map(b => b.text).join('\n') + - '\n[图片数据已省略:本图已随上一条回执发送过,不再重复携带;若你当时未能识别其内容,请直接告知用户,勿反复重读同一文件]'; + content = blocks.filter(b => b && b.type === 'text').map(b => b.text).join('\n'); + // 已消费的图重放时不再携带像素:把回执的 image_attached 降为 false 并给出原因, + // 模型读字段就知道「不是又丢了」,不会因叙事惯性重读。旧库散文回执保持原有尾巴。 + const consumed = patchImageReceipt(content, { + image_attached: false, reason: 'omitted_already_sent', retryable: false, + hint: '该图已在此前的轮次投递过,重放时不再重复携带像素;若你当时未能识别,请直接告知用户,勿反复重读同一文件', + }); + content = consumed !== null ? consumed + : content + '\n[图片数据已省略:本图已随上一条回执发送过,不再重复携带;若你当时未能识别其内容,请直接告知用户,勿反复重读同一文件]'; } else if (row.role === 'tool') { // 未消费的图:回执只留文字,图块挪进批次后的 user 合并消息(见 pendingImages/Labels) const texts = blocks.filter(b => b && b.type === 'text').map(b => b.text); content = texts.join('\n'); - const label = (texts[0] || 'read_file 图片回传').split('\n')[0]; + const label = imageLabel(texts[0]); + let mintFailed = false; for (const b of blocks) { if (!b || (b.type !== 'image_ref' && b.type !== 'image_url')) continue; const seq = ++imgSeq; @@ -287,9 +320,19 @@ function buildMessages(session, language, mintUrl, profile, withProfile, trimOn) pendingLabels.push('[图片 ' + seq + '] ' + label); pendingImages.push({ type: 'image_url', image_url: { url } }); } else { - pendingLabels.push('[图片 ' + seq + '] ' + label + '(外链不可用,未取得像素:请直接告知用户,勿反复重读同一文件)'); + mintFailed = true; + pendingLabels.push('[图片 ' + seq + '] ' + label + '(未送达:link_unavailable)'); } } + // 签不到外链 = 本次请求像素确实到不了:当场把回执字段降级,让模型拿到的是 + // 「ok 但 image_attached=false + reason」,而不是一句需要它自己解读的说明。 + if (mintFailed) { + const degraded = patchImageReceipt(content, { + image_attached: false, reason: 'link_unavailable', retryable: false, + hint: '服务端签不出图片外链,本张图的像素未能随本次请求送达;请直接告知用户,勿反复重读同一文件', + }); + if (degraded !== null) content = degraded; + } } else { // user 行(粘贴附件/旧形态):image_url 块(data: URL)原样;image_ref → 现签公网外链 //(每次请求重签,不过期) @@ -646,4 +689,4 @@ async function callModel(cfg, modelId, messages, tools, options) { return { content, reasoning, tool_calls: Array.from(toolCalls.values()), finish_reason: finishReason, usage, aborted, sseFrames }; } -module.exports = { callModel, buildMessages, buildSystemPrompt, systemPromptText, setSkillListText, setPayloadTrim, setProfilePrompts, encodeBlocks, decodeBlocks, compressMessages, titleMessages }; +module.exports = { callModel, buildMessages, buildSystemPrompt, systemPromptText, setSkillListText, setPayloadTrim, setProfilePrompts, encodeBlocks, decodeBlocks, compressMessages, titleMessages, patchImageReceipt }; diff --git a/app/server/templates/profile-default.txt b/app/server/templates/profile-default.txt index 90155b0..0eb1c82 100644 --- a/app/server/templates/profile-default.txt +++ b/app/server/templates/profile-default.txt @@ -95,8 +95,10 @@ When the user asks for something that seems out of scope: carry state across calls. - Python 3.11 is available; the system Python and its packages are read-only. To add packages, create a virtualenv inside the working directory - (`python3 -m venv .venv`, then `.venv/bin/pip` and `.venv/bin/python`). Keep - it in the working directory, not `/tmp`. + (`python3 -m venv --system-site-packages .venv`, then `.venv/bin/pip` and `.venv/bin/python`). + Keep it in the working directory, not `/tmp`. + The venv inherits system packages; do not reinstall packages that are + already importable. Only install what is missing. - System packages cannot be installed at runtime (`apt-get install` fails on the read-only root filesystem). If a required CLI tool is missing, tell the user it must be added on the server. diff --git a/app/server/tools/definitions.json b/app/server/tools/definitions.json index ed9987c..7acbd00 100644 --- a/app/server/tools/definitions.json +++ b/app/server/tools/definitions.json @@ -219,7 +219,7 @@ "type": "function", "function": { "name": "read_file", - "description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nCross-session peek: `path` accepts '@/' to read a text file from your OWN past session, read-only (find the id with recall op=search, browse with list_directory). It is a look, not an import: nothing lands in your session directory, so `terminal`, grep and `edit_file` cannot see it, and past images are refused. When you need to work with a file or view a past image, `copy_path` it first (its `source_path` takes the same '@' form). '@' is never accepted by `write_file` or `edit_file`.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF) are returned as visual content you can analyze directly, but only if the current model supports vision; otherwise a text placeholder is returned — that placeholder is not an error. The tool result itself carries a text notice only; the actual image pixels arrive in a separate user message right after the batch of tool results: that message lists per-image labels matching each image to its read_file receipt, then the image blocks in receipt order, and asks you to briefly describe each image before continuing. Never re-read an image already delivered in the current turn; if pixels never arrive, tell the user instead of retrying. Very large images (roughly over 50 MB) are skipped, and images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before being returned.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).", + "description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nCross-session peek: `path` accepts '@/' to read a text file from your OWN past session, read-only (find the id with recall op=search, browse with list_directory). It is a look, not an import: nothing lands in your session directory, so `terminal`, grep and `edit_file` cannot see it, and past images are refused. When you need to work with a file or view a past image, `copy_path` it first (its `source_path` takes the same '@' form). '@' is never accepted by `write_file` or `edit_file`.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF): the receipt is a single JSON object, not prose — read its fields instead of interpreting the text. Success shape: {\"ok\":true,\"tool\":\"read_file\",\"kind\":\"image\",\"path\":\"...\",\"image_attached\":true,\"width\":1080,\"height\":1920,\"format\":\"jpeg\",\"bytes_in\":3550000,\"bytes_out\":210000,\"resized\":true,\"source_width\":4000,\"source_height\":7111}. `width`/`height` are the delivered pixel size; `source_width`/`source_height` appear only when the image was downscaled. `image_attached:true` means the server produced the image and it will arrive as an image block in the separate user message that immediately follows the batch of tool results; that message labels each image `[图片 N] read_file ×` and asks you to describe it before continuing. The tool result itself never carries pixels. Refusal or failure shape: {\"ok\":false,\"kind\":\"image\",\"path\":\"...\",\"image_attached\":false,\"reason\":\"...\",\"retryable\":false,\"hint\":\"...\"} where `reason` is one of no_vision (this model cannot see images), too_large (roughly over 50 MB), past_session (import it with copy_path first), decode_failed, sharp_missing, already_delivered (the same image was already sent earlier in this turn), link_unavailable (no public URL could be minted, so the pixels did not arrive for that request), omitted_already_sent (an older receipt whose image was already delivered; replay omits the pixels). `ok:true` with `image_attached:false` is still a successful call, e.g. a deduplicated re-read. When `retryable` is false do not retry — tell the user instead. Images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before delivery.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).", "parameters": { "type": "object", "required": [ diff --git a/app/server/tools/executor.js b/app/server/tools/executor.js index 8e2b433..aa6ab5a 100644 --- a/app/server/tools/executor.js +++ b/app/server/tools/executor.js @@ -129,6 +129,9 @@ const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '. // read_file 读图(仅当会话绑定的模型 vision: true):sharp 压缩后落盘到任务目录 .imgcache/, // 返回 {text, image_rel};engine 拼成 image_ref 内容块落库,llm.buildMessages 发请求前 // 现签成公网外链(GET /p/)进 image_url 块,base64 不再进 DB/请求体。 +// text 一律是 imageReceipt 生成的 JSON 对象(字段契约见下),不是散文: +// 这份回执会被三层改写(executor 产出 / engine 同回合去重 / llm 签外链失败降级), +// 散文回执让模型只能靠语义猜「到底有没有图」,历史事故是反复 read_file 补看烧 token。 const IMG_MAX_INPUT_BYTES = 50 * 1024 * 1024; // 与上传上限一致 const IMG_MAX_SIDE = 2000; // 长边压缩阈值(px) const IMG_JPEG_QUALITY = 80; @@ -178,14 +181,51 @@ function outlineWrapper(outlineText) { 'For example, to read a function shown as [L100-150], use start_line: 100 and end_line: 150.'; } +// read_file 图片回执的统一信封。字段是模型唯一可判定的事实,回执不再写中文句子: +// ok 调用本身是否成功(ok=true + image_attached=false 是合法形态:同回合重复读被去重) +// path 模型请求的路径(@ 历史形原样保留);投递产物路径只在服务端内部用,不回执 +// image_attached 服务端是否产出了图块、会随本批回执之后的那条 user 消息投递(≠上游已收到) +// reason 仅 image_attached=false 时有值,稳定枚举(no_vision / too_large / past_session / +// decode_failed / sharp_missing / already_delivered / link_unavailable / +// omitted_already_sent)——模型按它分支,不要读 hint 猜 +// retryable 重试同一调用是否有意义(图片这条路上恒 false:同样的字节只会得到同样的结果) +// width/height 实际投递的像素尺寸;source_width/height + resized 只在确实缩放过时出现 +// hint 一句人话补充,仅供解释,不承载判定 +function imageReceipt(o) { + const out = { ok: o.ok === true, tool: 'read_file', kind: 'image', path: o.path, image_attached: o.image_attached === true }; + for (const k of ['reason', 'retryable', 'width', 'height', 'format', 'bytes_in', 'bytes_out', + 'resized', 'source_width', 'source_height', 'hint']) { + if (o[k] !== undefined) out[k] = o[k]; + } + return JSON.stringify(out); +} + // 用 sharp 压缩(按 EXIF 摆正、长边 ≤2000px、JPEG 质量 80)并落盘 .imgcache/。 -// 成功返回 {text, image_rel};产物文件名带源文件指纹(mtime+size),同图重复读复用同一份不增殖。 +// 成功返回 {text, image_rel},失败返回 {text}(text 都是 imageReceipt 的 JSON)。 +// 产物文件名带源文件指纹(mtime+size),同图重复读复用同一份不增殖。 // 无收益守卫(与浏览器端压缩同款):重编码 JPEG 反而 ≥ 原图时不落盘,image_rel 直指原文件。 -// sharp 缺失或解码失败时退回纯文本提示,不让整个工具回合报错。 +// sharp 缺失或解码失败时只回执原因,不让整个工具回合报错。 async function readImageAsBlocks(ctx, abs, rel, size) { let sharp; try { sharp = require('sharp'); } - catch (_) { return '[图片文件] ' + rel + '(' + size + ' 字节)。服务端未安装 sharp,无法压缩回传图片内容。'; } + catch (_) { + return { + text: imageReceipt({ + ok: false, path: rel, image_attached: false, reason: 'sharp_missing', retryable: false, bytes_in: size, + hint: '服务端未安装 sharp,无法压缩回传图片内容', + }), + }; + } + // 源尺寸只读文件头(metadata 不解码像素);EXIF orientation 5-8 表示显示时旋转 90°, + // 宽高要交换才与模型最终看到的画面一致(sharp 的 rotate() 会真的转)。 + let src = null; + try { + const m = await sharp(abs).metadata(); + if (m && m.width && m.height) { + const swap = Number(m.orientation) >= 5; + src = { w: swap ? m.height : m.width, h: swap ? m.width : m.height, format: m.format || null }; + } + } catch (_) { src = null; } try { const r = await sharp(abs) .rotate() @@ -195,7 +235,13 @@ async function readImageAsBlocks(ctx, abs, rel, size) { if (r.data.length >= size) { // 低内容熵图(截图/线稿/纯色)JPEG 重编码可能比原图还大:不产出缓存,外链直接指向原文件 return { - text: '[图片文件] ' + rel + '(' + size + ' 字节,压缩无收益,按原图回传)', + text: imageReceipt({ + ok: true, path: rel, image_attached: true, + width: src ? src.w : undefined, height: src ? src.h : undefined, + format: src && src.format ? src.format : undefined, + bytes_in: size, bytes_out: size, resized: false, + hint: '压缩无收益,按原图回传', + }), image_rel: rel, }; } @@ -208,13 +254,21 @@ async function readImageAsBlocks(ctx, abs, rel, size) { await fsp.mkdir(path.dirname(outAbs), { recursive: true }); await fsp.writeFile(outAbs, r.data); } - return { - text: '[图片文件] ' + rel + '(原 ' + size + ' 字节,已压缩为 ' + r.info.width + '×' + r.info.height + - ' JPEG ' + r.data.length + ' 字节)', - image_rel: cacheRel, + const receipt = { + ok: true, path: rel, image_attached: true, + width: r.info.width, height: r.info.height, format: 'jpeg', + bytes_in: size, bytes_out: r.data.length, + resized: src ? (r.info.width !== src.w || r.info.height !== src.h) : undefined, }; + if (src) { receipt.source_width = src.w; receipt.source_height = src.h; } + return { text: imageReceipt(receipt), image_rel: cacheRel }; } catch (err) { - return '[图片文件] ' + rel + '(' + size + ' 字节),图片解码/压缩失败:' + (err && err.message || err); + return { + text: imageReceipt({ + ok: false, path: rel, image_attached: false, reason: 'decode_failed', retryable: false, bytes_in: size, + hint: '图片解码/压缩失败:' + (err && err.message || err), + }), + }; } } @@ -244,15 +298,29 @@ async function toolReadFile(ctx, args) { // 不做「偷偷复制」:图片要出现在模型视野里,字节必须落到本会话才能签外链(engine 的 // mintImageUrl 与前端下载路由都以当前 taskDir 为根),那本质已是一次导入——与其藏进 // .imgcache 不进文件表,不如让模型明确知道历史图片必须先 copy_path。 - return '[图片文件] ' + disp + '(' + st.size + ' 字节)历史会话图片不支持直接读取。' + - '图片需要先 copy_path(source_path 用这个 @ 路径)导入本会话,再 read_file——导入后图片、terminal、mint 全量生效。'; + return { + text: imageReceipt({ + ok: false, path: disp, image_attached: false, reason: 'past_session', retryable: false, bytes_in: st.size, + hint: '历史会话图片不支持直接读取:先 copy_path(source_path 用这个 @ 路径)导入本会话,再 read_file——导入后图片、terminal、mint 全量生效', + }), + }; } const rel = relOf(ctx.taskDir, abs); if (!ctx.vision) { - return '[图片文件] ' + rel + '(' + st.size + ' 字节)。图片文件,当前模型不支持视觉识别,无法回传图片内容。'; + return { + text: imageReceipt({ + ok: false, path: rel, image_attached: false, reason: 'no_vision', retryable: false, bytes_in: st.size, + hint: '当前模型不支持视觉识别,无法回传图片像素;请基于文件名/大小作答,或直接告知用户', + }), + }; } if (st.size > IMG_MAX_INPUT_BYTES) { - return '[图片文件] ' + rel + '(' + st.size + ' 字节)过大(上限 ' + IMG_MAX_INPUT_BYTES + ' 字节),已跳过回传。'; + return { + text: imageReceipt({ + ok: false, path: rel, image_attached: false, reason: 'too_large', retryable: false, bytes_in: st.size, + hint: '超过回传上限 ' + IMG_MAX_INPUT_BYTES + ' 字节;可先用 terminal 缩小(如 ImageMagick)再读', + }), + }; } return await readImageAsBlocks(ctx, abs, rel, st.size); } @@ -1660,7 +1728,8 @@ const HANDLERS = { }; // ctx: {sessionId, taskDir, run, cfg, db, vision} -// 返回值一般为字符串;read_file 读图且 ctx.vision 为 true 时返回 {text, image_rel}。 +// 返回值一般为字符串;read_file 读图时返回 {text, image_rel},text 是结构化 JSON 回执 +// (见 imageReceipt),image_rel 缺省表示本次不投递像素。 // 内置工具优先;未命中时交给 MCP 注册表(启动时发现的服务工具)。 async function executeTool(baseCtx, name, args) { const handler = HANDLERS[name]; diff --git a/docs/tool-design.md b/docs/tool-design.md index 8e2adcd..d06d597 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -65,3 +65,13 @@ system prompt 不重复 schema 已表达的内容。 ### 八、上下文成本 tools 数组每轮请求完整重发。新工具进来时,先问"能不能装进已有工具",再问"要不要新建"。 + +### 九、回执形态 + +- **内容型**工具(`read_file` 的文本、`fetch` 的正文、`db` 的查询结果)回执就是内容本身,不要套 JSON 信封。 +- **结果型**动作(读图、裁剪、转换)回执用**单个 JSON 对象**,把判定写进字段,不写自然语言:模型按字段分支, + 不要再让它从句子里猜。字段名跨工具保持一致:`ok`(调用是否成功)/ `reason`(稳定枚举)/ `retryable` / + `width` `height` 之类的事实字段 / `hint`(仅解释,不承载判定)。 +- 回执可能被多层改写(发送前裁剪、同回合去重、投递失败降级):改写**只动字段**,不得在 JSON 后追加句子—— + 追加会把回执变成不可解析的尾巴,等于把判定权又还给语义猜测。 +- `ok:true` 不等于「结果已送达」:另有字段(如 `image_attached`)描述投递状态,两者不要混成一个布尔。