diff --git a/app/package-lock.json b/app/package-lock.json index 69a4186..c7159a9 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,18 +1,19 @@ { "name": "agent-app", - "version": "2.1.0", + "version": "2.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-app", - "version": "2.1.0", + "version": "2.3.0", "dependencies": { "better-sqlite3": "11.3.0", "cron-parser": "5.10.1", "express": "4.21.2", "multer": "1.4.5-lts.1", "sharp": "0.35.4", + "turndown": "7.2.0", "yaml": "2.6.1" } }, @@ -526,6 +527,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "http://mirrors.cloud.tencent.com/npm/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "http://mirrors.tencentyun.com/npm/accepts/-/accepts-1.3.8.tgz", @@ -1929,6 +1936,15 @@ "node": "*" } }, + "node_modules/turndown": { + "version": "7.2.0", + "resolved": "http://mirrors.cloud.tencent.com/npm/turndown/-/turndown-7.2.0.tgz", + "integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "http://mirrors.tencentyun.com/npm/type-is/-/type-is-1.6.18.tgz", diff --git a/app/package.json b/app/package.json index afbb0dd..2b0abb3 100644 --- a/app/package.json +++ b/app/package.json @@ -13,6 +13,7 @@ "express": "4.21.2", "multer": "1.4.5-lts.1", "sharp": "0.35.4", + "turndown": "7.2.0", "yaml": "2.6.1" } } diff --git a/app/server/tools/executor.js b/app/server/tools/executor.js index 70cdac3..04dfcfe 100644 --- a/app/server/tools/executor.js +++ b/app/server/tools/executor.js @@ -9,6 +9,7 @@ const { spawn } = require('child_process'); const mcp = require('../mcp'); const skills = require('./skills'); const Database = require('better-sqlite3'); // db 工具:会话目录下的 data.db(同步 API,与单进程模型契合) +const TurndownService = require('turndown'); // fetch 工具:HTML→Markdown 用真解析器(内置 htmlparser2,容错截断/畸形文档) const rexec = require('../remote-exec'); const publicFiles = require('../public-files'); const scheduler = require('../scheduler'); // 只依赖 log/cron-parser,不成环(engine→executor→scheduler 单向) @@ -804,7 +805,7 @@ function fetchCacheSet(run, key, entry) { function fetchWindow(text, off, maxOut, hard, truncated, maxBytes) { if (off >= text.length) { return '(offset=' + off + ' 超出全文长度 ' + text.length + ' 字符,无内容可读)' + - (truncated ? '\n(源响应体超过抓取上限 ' + maxBytes + ' 字节,可用更大 max_bytes 重抓)' : ''); + (truncated ? '\n(源响应体在 ' + maxBytes + ' 字节处截断,正文尾部缺失;可用更大 max_bytes 重抓)' : ''); } const end = Math.min(text.length, off + maxOut); let out = text.slice(off, end); @@ -812,7 +813,7 @@ function fetchWindow(text, off, maxOut, hard, truncated, maxBytes) { out += '\n\n(已截断:全文 ' + text.length + ' 字符,本段 ' + off + '–' + end + '。' + '续读传 offset=' + end + '(不重新下载);单段可传 max_chars 放大,硬帽 ' + hard + ')'; } - if (truncated) out += '\n(源响应体超过抓取上限 ' + maxBytes + ' 字节,内容末尾可能不完整)'; + if (truncated) out += '\n(源响应体在 ' + maxBytes + ' 字节处截断,正文尾部缺失,并非解析错误;传更大 max_bytes 重抓可拿完整正文)'; return out; } @@ -920,7 +921,15 @@ async function toolFetch(ctx, args) { try { text = new TextDecoder(charset, { fatal: false }).decode(buf); } catch (_) { text = buf.toString('utf8'); } } - let body = ct.includes('html') ? htmlToMarkdown(text) : text; + let body; + if (ct.includes('html')) { + // DOM 树内存约为原始字节的 3–5 倍:超解析硬帽不再做 DOM 解析,降级轻量剥标签——脏但别空手而归。 + body = buf.length > HTML_PARSE_HARD_CAP + ? '[响应体超解析上限 ' + HTML_PARSE_HARD_CAP + ' 字节,已降级为剥标签纯文本]\n\n' + fallbackStripHtml(text) + : htmlToMarkdown(text, cur); // base=重定向终址:链接绝对化的根 + } else { + body = text; + } // 字节截断可能切在多字节字符中间,尾部的 U+FFFD 是噪音,去掉。 if (truncated) body = body.replace(/\uFFFD+$/, ''); const win = fetchWindow(body, off, maxOut, hard, truncated, maxBytes); @@ -944,63 +953,92 @@ async function toolFetch(ctx, args) { } } -// 单遍实体解码:旧版链式 replace 会把 < 双重解码成 <;HTML5 全量表 2000+ 条, -// 这里只收网页文本真正常见的命名实体,数字实体(十进制/十六进制)通吃,认不出的原样保留。 -const HTML_ENTITIES = { - nbsp: ' ', ensp: '\u2002', emsp: '\u2003', thinsp: '\u2009', zwnj: '\u200c', zwj: '\u200d', shy: '\u00ad', - amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", - copy: '\u00a9', reg: '\u00ae', trade: '\u2122', deg: '\u00b0', hellip: '\u2026', - mdash: '\u2014', ndash: '\u2013', lsquo: '\u2018', rsquo: '\u2019', sbquo: '\u2018', - ldquo: '\u201c', rdquo: '\u201d', bdquo: '\u201e', laquo: '\u00ab', raquo: '\u00bb', - lsaquo: '\u2039', rsaquo: '\u203a', prime: '\u2032', Prime: '\u2033', bull: '\u2022', middot: '\u00b7', - times: '\u00d7', divide: '\u00f7', plusmn: '\u00b1', frac12: '\u00bd', frac14: '\u00bc', frac34: '\u00be', - sup1: '\u00b9', sup2: '\u00b2', sup3: '\u00b3', micro: '\u00b5', para: '\u00b6', sect: '\u00a7', - dagger: '\u2020', Dagger: '\u2021', permil: '\u2030', - euro: '\u20ac', pound: '\u00a3', yen: '\u00a5', cent: '\u00a2', curren: '\u00a4', - agrave: '\u00e0', aacute: '\u00e1', acirc: '\u00e2', atilde: '\u00e3', auml: '\u00e4', aring: '\u00e5', ccedil: '\u00e7', - egrave: '\u00e8', eacute: '\u00e9', ecirc: '\u00ea', euml: '\u00eb', - igrave: '\u00ec', iacute: '\u00ed', icirc: '\u00ee', iuml: '\u00ef', - ograve: '\u00f2', oacute: '\u00f3', ocirc: '\u00f4', otilde: '\u00f5', ouml: '\u00f6', - ugrave: '\u00f9', uacute: '\u00fa', ucirc: '\u00fb', uuml: '\u00fc', - yacute: '\u00fd', yuml: '\u00ff', ntilde: '\u00f1', szlig: '\u00df', - ne: '\u2260', le: '\u2264', ge: '\u2265', infin: '\u221e', sup: '\u2283', sub: '\u2286', -}; -function decodeEntities(s) { - return s.replace(/([0-9a-f]+);|(\d+);|&([a-zA-Z][a-zA-Z0-9]*);/gi, (m, hx, dec, name) => { - try { - if (hx) return String.fromCodePoint(parseInt(hx, 16)); - if (dec) return String.fromCodePoint(parseInt(dec, 10)); - // hasOwnProperty:防 &constructor; 这类名字踩到原型链。 - return Object.prototype.hasOwnProperty.call(HTML_ENTITIES, name) ? HTML_ENTITIES[name] : m; - } catch (_) { return m; } // 码点越界等原样保留 +// ---------- HTML → Markdown 转换(turndown:真 DOM 解析器) ---------- +// 为什么不再用正则剥标签:截断的 HTML 让正则方案必然漏——缺 时 [\s\S]*?<\/\1> +// 整体失配,半截 JS 源码会当正文漏出;未闭合的开标签残片同理穿过 <[^>]+> 掉进正文。 +// turndown 内置 htmlparser2 按容错规则建树:EOF 前的残文统一归进 script 节点、随节点丢弃。 +// 剥壳/标题/链接绝对化全走 DOM 谓词,不碰字符串;实体解码由解析器完成(decodeEntities 表删)。 +// DOM 树内存约为原始字节的 3–5 倍:响应体超解析硬帽时不做 DOM 解析,降级轻量剥标签(脏但不空手而归)。 +const HTML_PARSE_HARD_CAP = 10 * 1024 * 1024; + +// 页面外壳标签:正文密度低于信噪比门槛,整块丢。head 不在列——title 子节点还要靠它产出 +// 首行标题,其余 head 子节点(meta/link/base/style/script)已各自在列。 +const HTML_CHROME_TAGS = new Set([ + 'script', 'style', 'noscript', 'svg', 'template', 'nav', 'header', 'footer', 'aside', + 'form', 'iframe', 'button', 'input', 'select', 'textarea', 'label', 'meta', 'link', 'base', +]); +// class/id 启发式限定块级容器:不限标签会连坐删掉
这类正文。 +// 子串匹配保留旧版口径(navbar/comments/ad-slot 等变体都命中),接受误杀换召回。 +const HTML_CHROME_CONTAINERS = new Set(['div', 'section', 'article', 'ul', 'ol', 'main']); +const HTML_CHROME_CLASS_ID = + /nav|menu|sidebar|breadcrumb|comment|footer|related|banner|promo|advert|popup|share|subscribe|masthead|cookie|social|toolbar/i; + +// turndown 的 htmlparser2 不规格化属性名大小写(CLASS= 在 attribs 里就是 CLASS),且 domhandler +// 的 getAttribute 按精确键查 → 大小写不敏感地扫一遍(标签名不受影响,解析时已统一小写)。 +function domAttr(node, name) { + const a = node.attribs || {}; + if (a[name] != null) return a[name]; + for (const k in a) { if (k.toLowerCase() === name) return a[k]; } + return ''; +} + +// 每次抓取新建实例:链接绝对化要把本次重定向后的终址闭进规则,共享单例会被并发抓取互踩; +// 构造成本可忽略,解析才是大头。 +function makeTurndown(baseUrl) { + const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', hr: '---', bulletListMarker: '-' }); + td.remove((node) => { + if (!node || node.nodeType !== 1) return false; + const tag = String(node.nodeName).toLowerCase(); + if (HTML_CHROME_TAGS.has(tag)) return true; + if (!HTML_CHROME_CONTAINERS.has(tag)) return false; + return HTML_CHROME_CLASS_ID.test(domAttr(node, 'class') + ' ' + domAttr(node, 'id')); }); + //