多模态投递定案:探针矩阵证伪「每轮一张」限流与外链嫌疑,合并消息按官方形态收敛——DeepSeek 视觉受控实验(纯色条 ground-truth 图 × prompt_tokens 差值 × /p 外部取用日志,三源交叉):data URL 单/双图、/p 外链单/双图(自签 https+attachment 头)、tool 回执后 user 合并形态、真实微信截图逐字 OCR,9 项全 PASS;生产 DB usage.pt 实测每图轮 +600~1000——14:33 时图早已进入上下文,"看不见"是弱模型叙事惯性而非投递失败。据此采纳评审可落地三条:llm.js 合并 user 消息改 text 置顶排布(逐图对应说明合并进首段文字、图块按回执顺序紧随,对齐官方示例形态),首段明说「不是用户新输入」并附「先逐张简述再继续」指令兼作送达自检;"已消费"分界从任意 assistant 收紧为最后一条无 tool_calls 的收尾 assistant(中间只发 tool_calls 的轮次不算已阅,图继续留在上下文直到模型真正答复过,鲁棒优先于极致省 token)。engine.js 撤 imgThisTurn 每轮一张批次限流(多图实测可行,该限流是为误诊买的保险),保留同回合同图去重护栏(本轮实测 11:51 式 40 连读已收敛为 1 次挡回+如实上报);「发送形态」debug 日志转常驻观测。definitions.json read_file 图片段与 ARCHITECTURE 图片链路、陷阱 7 同步(含诊断三源方法论:发送形态日志/外部取用行/pt 增量)。探针实验产物已全部清理;docker-compose.yml 为用户自改不在本次提交。

This commit is contained in:
2026-09-28 16:27:00 +08:00
parent 85ade78cec
commit b4d39675ec
4 changed files with 41 additions and 28 deletions
+3 -11
View File
@@ -566,7 +566,6 @@ async function runLoopInner(sessionId, run) {
const txt = String(m.content || '');
return m.role + (m.tool_calls ? '[calls]' : '')
+ (txt.includes('图片数据已省略') ? '[图已省]' : '')
+ (txt.includes('每轮只能携带一张图片') ? '[图被限流]' : '')
+ (txt.includes('本回合已投递过') ? '[图重复读挡回]' : '');
}).join(' '));
}
@@ -677,8 +676,6 @@ async function runLoopInner(sessionId, run) {
// 执行工具
const vision = modelVision(session);
let handoffDone = false; // 本批工具里是否有 handoff 成功返回(批量跑完统一切换,见循环尾注释)
// 本批(=本次模型调用可见范围)是否已放行过一张图;随 while 每轮迭代重置。
let imgThisTurn = false;
for (const tc of msgRow.tool_calls) {
if (run.cancelled) break;
const id = tc.id;
@@ -720,21 +717,16 @@ async function runLoopInner(sessionId, run) {
outText = String(output == null ? '' : output);
}
if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)';
// 死循环护栏(实测:多图同时进上下文时部分上游只认一张;叠加催读文案会让模型整轮重读同一批图、
// 每轮都签新图烧 token 直到迭代耗尽):
// ① 每次模型调用最多携带一张图:同批后续图回执降级为文字,让模型下一轮单独再读,逐轮收敛;
// ② 同回合同一张图只投递一次:已投递过的再读只回文字提示(run 随回合销毁,新回合自动重置额度)。
// 死循环护栏:同回合内同一张图只投递一次——上游能认多图(探针实测同消息双外链全对),
// 不再按批次限流;但模型若因惯性叙事反复重读同一文件,每轮都签新图就是纯烧 token,
// 故已投递过的图再读只回文字提示(run 随回合销毁,新回合自动重置,追问重看不受影响)。
if (imageRel) {
if (!run.imgDelivered) run.imgDelivered = new Set();
if (run.imgDelivered.has(imageRel)) {
outText += '\n[该图片本回合已投递过,不再重复携带;若你未看到它,请直接告知用户,勿反复重读同一文件]';
imageRel = null;
} else if (imgThisTurn) {
outText += '\n[每轮只能携带一张图片,本张未投递:请在下一轮单独 read_file 本文件查看]';
imageRel = null;
} else {
run.imgDelivered.add(imageRel);
imgThisTurn = true;
}
}
log.debug('tool', sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText);
+35 -14
View File
@@ -128,9 +128,13 @@ function cachedDecodeBlocks(session, row) {
// 大量 OpenAI 兼容上游会整批忽略或只认批次最后一张,模型看不到图却见回执写着
// 「已回传」,会反复 read_file 补看(死循环烧 token,见多模态回退复盘);
// 故回执降级为纯文本,图块现签成 image_url(mintUrl 公网链回调)后统一并入
// 该批回执之后的一条 user 消息(user 角色各家通吃),每张图前配一行对应回执文字;
// 该批之后的一条 user 消息(user 角色各家通吃):逐图说明文字全部置顶、图块按回执顺序紧随
//(对齐官方示例排布;探针实测 DS 对交替排布也兼容,取低风险形态),
// 并附「先逐张简述再继续任务」指令——兼作送达自检,破弱模型的“没收到”惯性叙事;
// 签不到(base_url 未配等)降级为文字占位,绝不把 image_ref 这种非标准块发上游。
// 省 token:图块仅在其位于最后一条 assistant 消息之后(模型尚未对它回复)时携带;
// 省 token:图块保留到最后一条「收尾回复」(无 tool_calls 的 assistant)之前;
// 中间轮只发 tool_calls 不算看过,图继续携带——14:33 实测 pt 证明图已进上下文而模型
// 仍自述没看到(弱模型行为不可靠),行为鲁棒优先于极致省钱;更早的视为已消费,
// 之前的视为已消费,发请求时替换为文字标记(措辞不再催促重读)。DB 始终存全量:
// UI 历史照常渲染图片,模型如需重看可再次 read_file(新回执在尾部,会重新携带图片)。
// 压缩摘要相关行(见 engine.runLoopInner / summarizeContext):
@@ -162,10 +166,13 @@ function buildMessages(session, language, mintUrl, profile, withProfile) {
// tool 回执按 tool_call_id 索引:给 errAck 行拼错误提示时取执行时挡下的原因
const toolRowById = new Map();
for (const row of history) if (row.role === 'tool' && row.tool_call_id) toolRowById.set(row.tool_call_id, row);
// 「已消费」分界 = 最后一条给出收尾回复(无 tool_calls)的 assistant:模型只发
// tool_calls 的中间轮不算已阅图,图继续随请求携带直到它真正答复过一次。
// errAck / archived 不发模型,不算回复:它们之前的图不算被消费。
let lastAssistant = -1;
for (let i = history.length - 1; i >= 0; i--) {
// errAck / archived 不发模型,不算「模型已回复」:它们之前的图不算被消费,下次仍携带
if (history[i].role === 'assistant' && history[i].type !== 'errAck' && history[i].type !== 'archived') { lastAssistant = i; break; }
const r = history[i];
if (r.role === 'assistant' && r.type !== 'errAck' && r.type !== 'archived' && !r.tool_calls) { lastAssistant = i; break; }
}
const dropToolCallIds = new Set();
// 旧版 summarize_request 「已答复」判定的前置信息:最后一次「回合收尾回复」(无 tool_calls 的 assistant)的位置。
@@ -178,7 +185,8 @@ function buildMessages(session, language, mintUrl, profile, withProfile) {
// 未消费图片的暂存区:tool 回执降级为纯文本,图块收集到这里,
// 循环结束后合并成一条 user 消息发送(顺序约束:天然落在该批全部回执之后、
// 且是模型下一次的直接输入,assistant→tool 回执的紧邻约束不被打断)。
const pendingImages = [];
const pendingImages = []; // 待投递的图块(按回执顺序;签不到时不占位)
const pendingLabels = []; // 每图一行「对应回执」说明,合并进首条文字块(text 置顶、图在后)
let imgSeq = 0;
history.forEach((row, i) => {
// archived:UI 可见,但不进入模型上下文(节省 token)
@@ -226,17 +234,25 @@ function buildMessages(session, language, mintUrl, profile, withProfile) {
content = blocks.filter(b => b && b.type === 'text').map(b => b.text).join('\n') +
'\n[图片数据已省略:本图已随上一条回执发送过,不再重复携带;若你当时未能识别其内容,请直接告知用户,勿反复重读同一文件]';
} else if (row.role === 'tool') {
// 未消费的图:回执只留文字,图块挪进批次后的 user 合并消息(见 pendingImages)
// 未消费的图:回执只留文字,图块挪进批次后的 user 合并消息(见 pendingImages/Labels)
const texts = blocks.filter(b => b && b.type === 'text').map(b => b.text);
content = texts.join('\n');
const label = (texts[0] || 'read_file 图片回传').split('\n')[0];
for (const b of blocks) {
if (!b || (b.type !== 'image_ref' && b.type !== 'image_url')) continue;
// 每张图配一行对应回执说明(回执首行含文件名与压缩信息),模型能对上号
pendingImages.push({ type: 'text', text: '[图片 ' + (++imgSeq) + ',对应回执] ' + ((texts[0] || 'read_file 图片回传').split('\n')[0]) });
if (b.type === 'image_url') { pendingImages.push(b); continue; } // 旧库内联 data: URL 原样
const seq = ++imgSeq;
if (b.type === 'image_url') { // 旧库内联 data: URL 原样
pendingLabels.push('[图片 ' + seq + '] ' + label);
pendingImages.push(b);
continue;
}
const url = mintUrl ? mintUrl(b.rel) : null;
pendingImages.push(url ? { type: 'image_url', image_url: { url } }
: { type: 'text', text: '[图片外链不可用:未能取得图片内容,请直接告知用户,勿反复重读同一文件]' });
if (url) {
pendingLabels.push('[图片 ' + seq + '] ' + label);
pendingImages.push({ type: 'image_url', image_url: { url } });
} else {
pendingLabels.push('[图片 ' + seq + '] ' + label + '(外链不可用,未取得像素:请直接告知用户,勿反复重读同一文件)');
}
}
} else {
// user 行(粘贴附件/旧形态):image_url 块(data: URL)原样;image_ref → 现签公网外链
@@ -254,11 +270,16 @@ function buildMessages(session, language, mintUrl, profile, withProfile) {
messages.push({ role: row.role, content });
}
});
// 该批未消费图片合并发送:一条 user 消息带全部图块,逐图前置对应回执标签
if (pendingImages.length) {
// 该批未消费图片合并发送:单条 text 置顶列出逐图对应说明,图块按编号顺序紧随;
// 「先简述每张图再继续」兼作送达自检——模型描述得出来 = 像素真到了;描述不出 = 真没到,
// 不会再像 14:33 那样带着图却沿“没收到”的叙事惯性重读。
if (pendingImages.length || pendingLabels.length) {
messages.push({
role: 'user',
content: [{ type: 'text', text: '[系统] 以上工具回执对应的图片内容如下(read_file 回传,按回执顺序):' }]
content: [{ type: 'text',
text: '[系统提示] 以下不是用户新输入,是上面这批 tool 回执对应的图片(read_file 回传):\n'
+ pendingLabels.join('\n')
+ (pendingImages.length ? '\n请按编号顺序逐张简述每张图片里有什么(表明像素确实送达),然后继续完成用户的任务。' : '') }]
.concat(pendingImages),
});
}
+1 -1
View File
@@ -202,7 +202,7 @@
"type": "function",
"function": {
"name": "read_file",
"description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF) are returned as visual content you can analyze directly, but only if the current model supports vision; otherwise a text placeholder is returned — that placeholder is not an error. The tool result itself carries a text notice only; the actual image pixels arrive in a separate user message right after the batch of tool results, each image preceded by a label naming the read_file result it belongs to. At most ONE image arrives per round: when several images are read in one batch only the first carries pixels and the rest come back as text asking you to read_file them individually next round — so read images one at a time when you need several, never re-read an image already delivered in this turn, and if pixels never arrive tell the user instead of retrying. Very large images (roughly over 50 MB) are skipped, and images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before being returned.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).",
"description": "Reads the content of a file.\n\nText files: each returned line is prefixed with a line number right-aligned in a 6-character field, followed by a tab, then the line content.\n\nLarge text files (roughly over 16 KB) are not returned in full: you get an outline listing top-level symbols (or Markdown headings) with line numbers instead. The outline IS a successful response — do NOT retry the same call without a line range; use the outline line numbers with `start_line` / `end_line` to read specific sections. If no outline can be extracted, a first-1KB excerpt is returned instead. Supplying `start_line` or `end_line` disables outline mode.\n\nImage files (PNG, JPEG, WebP, GIF, BMP, TIFF) are returned as visual content you can analyze directly, but only if the current model supports vision; otherwise a text placeholder is returned — that placeholder is not an error. The tool result itself carries a text notice only; the actual image pixels arrive in a separate user message right after the batch of tool results: that message lists per-image labels matching each image to its read_file receipt, then the image blocks in receipt order, and asks you to briefly describe each image before continuing. Never re-read an image already delivered in the current turn; if pixels never arrive, tell the user instead of retrying. Very large images (roughly over 50 MB) are skipped, and images may be downscaled (long edge ~2000 px, re-encoded as JPEG) before being returned.\n\nBinary files are not returned as text; a short notice is returned instead. Text files over 32 MB cannot be read by this tool — use `terminal` (grep -n / head / sed -n).",
"parameters": {
"type": "object",
"required": [