refactor(tools): fetch 的 HTML 转换换 turndown DOM 解析
- 截断/畸形 HTML 不再漏残标签与半截 script 正文(正则 [\s\S]*? 遇缺闭标签整体失配的必然缺陷) - class/id 页壳启发式限定块级容器,不再连坐删正文 - 相对 href/src 按重定向终址绝对化,抓回的链接可直接作为下一次 fetch 参数 - 标题走 DOM 规则取(修复先删 head 后取 title 会取空的问题) - 新增解析硬帽 10MB:超限与解析抛错均降级轻量剥标签,不空手而归 - 截断提示文案区分「下载少了」与「解析坏了」
This commit is contained in:
Generated
+18
-2
@@ -1,18 +1,19 @@
|
||||
{
|
||||
"name": "agent-app",
|
||||
"version": "2.1.0",
|
||||
"version": "2.3.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "agent-app",
|
||||
"version": "2.1.0",
|
||||
"version": "2.3.0",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "11.3.0",
|
||||
"cron-parser": "5.10.1",
|
||||
"express": "4.21.2",
|
||||
"multer": "1.4.5-lts.1",
|
||||
"sharp": "0.35.4",
|
||||
"turndown": "7.2.0",
|
||||
"yaml": "2.6.1"
|
||||
}
|
||||
},
|
||||
@@ -526,6 +527,12 @@
|
||||
"url": "https://opencollective.com/libvips"
|
||||
}
|
||||
},
|
||||
"node_modules/@mixmark-io/domino": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "http://mirrors.cloud.tencent.com/npm/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
||||
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "http://mirrors.tencentyun.com/npm/accepts/-/accepts-1.3.8.tgz",
|
||||
@@ -1929,6 +1936,15 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "http://mirrors.cloud.tencent.com/npm/turndown/-/turndown-7.2.0.tgz",
|
||||
"integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mixmark-io/domino": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "http://mirrors.tencentyun.com/npm/type-is/-/type-is-1.6.18.tgz",
|
||||
|
||||
Reference in New Issue
Block a user