加入多模态图片识别功能
This commit is contained in:
@@ -12,6 +12,7 @@
|
|||||||
- 任务状态开关(规划 / 执行)
|
- 任务状态开关(规划 / 执行)
|
||||||
- 排队机制,用户可离线输入
|
- 排队机制,用户可离线输入
|
||||||
- 工具调用(文件操作、终端、网络抓取、网页搜索)
|
- 工具调用(文件操作、终端、网络抓取、网页搜索)
|
||||||
|
- 多模态图片识别(模型配置 vision: true 时,read_file 读图会压缩回传图片内容;用户上传仅存任务目录,不自动发送)
|
||||||
- 借鉴 Zed agent 的 system prompt 和工具定义
|
- 借鉴 Zed agent 的 system prompt 和工具定义
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ models:
|
|||||||
display_name: "AMD DeepSeek 1"
|
display_name: "AMD DeepSeek 1"
|
||||||
url: "https://proxy-a.example.com/v1" # 独立的 OpenAI 兼容 /v1 地址
|
url: "https://proxy-a.example.com/v1" # 独立的 OpenAI 兼容 /v1 地址
|
||||||
api_key: "sk-..." # 各模型可不同,留空则不带鉴权头
|
api_key: "sk-..." # 各模型可不同,留空则不带鉴权头
|
||||||
# vision: true # 选填,默认 false;支持图片时会话上下文会告知 AI
|
# vision: true # 选填,默认 false;支持时 read_file 读图会压缩回传(长边≤2000px JPEG),会话上下文会告知 AI
|
||||||
- id: ds2
|
- id: ds2
|
||||||
display_name: "AMD DeepSeek 2"
|
display_name: "AMD DeepSeek 2"
|
||||||
url: "https://proxy-b.example.com/v1"
|
url: "https://proxy-b.example.com/v1"
|
||||||
|
|||||||
Generated
+569
-2
@@ -1,20 +1,531 @@
|
|||||||
{
|
{
|
||||||
"name": "agent-app",
|
"name": "agent-app",
|
||||||
"version": "1.0.0",
|
"version": "2.0.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "agent-app",
|
"name": "agent-app",
|
||||||
"version": "1.0.0",
|
"version": "2.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"better-sqlite3": "11.3.0",
|
"better-sqlite3": "11.3.0",
|
||||||
"express": "4.21.2",
|
"express": "4.21.2",
|
||||||
"minimatch": "9.0.5",
|
"minimatch": "9.0.5",
|
||||||
"multer": "1.4.5-lts.1",
|
"multer": "1.4.5-lts.1",
|
||||||
|
"sharp": "0.35.4",
|
||||||
"yaml": "2.6.1"
|
"yaml": "2.6.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@emnapi/runtime": {
|
||||||
|
"version": "1.11.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@emnapi/runtime/-/runtime-1.11.3.tgz",
|
||||||
|
"integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"dependencies": {
|
||||||
|
"tslib": "^2.4.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/colour": {
|
||||||
|
"version": "1.1.0",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/colour/-/colour-1.1.0.tgz",
|
||||||
|
"integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-darwin-arm64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-darwin-arm64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-darwin-x64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-darwin-x64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-freebsd-wasm32": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"freebsd"
|
||||||
|
],
|
||||||
|
"dependencies": {
|
||||||
|
"@img/sharp-wasm32": "0.35.4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-darwin-arm64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-darwin-x64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-arm": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==",
|
||||||
|
"cpu": [
|
||||||
|
"arm"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-arm64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-ppc64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==",
|
||||||
|
"cpu": [
|
||||||
|
"ppc64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-riscv64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==",
|
||||||
|
"cpu": [
|
||||||
|
"riscv64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-s390x": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==",
|
||||||
|
"cpu": [
|
||||||
|
"s390x"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linux-x64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linuxmusl-arm64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-libvips-linuxmusl-x64": {
|
||||||
|
"version": "1.3.3",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz",
|
||||||
|
"integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-arm": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==",
|
||||||
|
"cpu": [
|
||||||
|
"arm"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-arm": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-arm64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-arm64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-ppc64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==",
|
||||||
|
"cpu": [
|
||||||
|
"ppc64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-ppc64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-riscv64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==",
|
||||||
|
"cpu": [
|
||||||
|
"riscv64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-riscv64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-s390x": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==",
|
||||||
|
"cpu": [
|
||||||
|
"s390x"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-s390x": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linux-x64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linux-x64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linuxmusl-arm64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linuxmusl-arm64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-linuxmusl-x64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-libvips-linuxmusl-x64": "1.3.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-wasm32": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==",
|
||||||
|
"license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
|
||||||
|
"optional": true,
|
||||||
|
"dependencies": {
|
||||||
|
"@emnapi/runtime": "^1.11.3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-webcontainers-wasm32": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==",
|
||||||
|
"cpu": [
|
||||||
|
"wasm32"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"optional": true,
|
||||||
|
"dependencies": {
|
||||||
|
"@img/sharp-wasm32": "0.35.4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-win32-arm64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0 AND LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-win32-ia32": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==",
|
||||||
|
"cpu": [
|
||||||
|
"ia32"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0 AND LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": "^20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@img/sharp-win32-x64": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "Apache-2.0 AND LGPL-3.0-or-later",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/accepts": {
|
"node_modules/accepts": {
|
||||||
"version": "1.3.8",
|
"version": "1.3.8",
|
||||||
"resolved": "http://mirrors.tencentyun.com/npm/accepts/-/accepts-1.3.8.tgz",
|
"resolved": "http://mirrors.tencentyun.com/npm/accepts/-/accepts-1.3.8.tgz",
|
||||||
@@ -1151,6 +1662,55 @@
|
|||||||
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
|
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
|
||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
|
"node_modules/sharp": {
|
||||||
|
"version": "0.35.4",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/sharp/-/sharp-0.35.4.tgz",
|
||||||
|
"integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@img/colour": "^1.1.0",
|
||||||
|
"detect-libc": "^2.1.2",
|
||||||
|
"semver": "^7.8.5"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.9.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://opencollective.com/libvips"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@img/sharp-darwin-arm64": "0.35.4",
|
||||||
|
"@img/sharp-darwin-x64": "0.35.4",
|
||||||
|
"@img/sharp-freebsd-wasm32": "0.35.4",
|
||||||
|
"@img/sharp-libvips-darwin-arm64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-darwin-x64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-arm": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-arm64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-ppc64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-riscv64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-s390x": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linux-x64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linuxmusl-arm64": "1.3.3",
|
||||||
|
"@img/sharp-libvips-linuxmusl-x64": "1.3.3",
|
||||||
|
"@img/sharp-linux-arm": "0.35.4",
|
||||||
|
"@img/sharp-linux-arm64": "0.35.4",
|
||||||
|
"@img/sharp-linux-ppc64": "0.35.4",
|
||||||
|
"@img/sharp-linux-riscv64": "0.35.4",
|
||||||
|
"@img/sharp-linux-s390x": "0.35.4",
|
||||||
|
"@img/sharp-linux-x64": "0.35.4",
|
||||||
|
"@img/sharp-linuxmusl-arm64": "0.35.4",
|
||||||
|
"@img/sharp-linuxmusl-x64": "0.35.4",
|
||||||
|
"@img/sharp-webcontainers-wasm32": "0.35.4",
|
||||||
|
"@img/sharp-win32-arm64": "0.35.4",
|
||||||
|
"@img/sharp-win32-ia32": "0.35.4",
|
||||||
|
"@img/sharp-win32-x64": "0.35.4"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"@types/node": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/side-channel": {
|
"node_modules/side-channel": {
|
||||||
"version": "1.1.1",
|
"version": "1.1.1",
|
||||||
"resolved": "http://mirrors.tencentyun.com/npm/side-channel/-/side-channel-1.1.1.tgz",
|
"resolved": "http://mirrors.tencentyun.com/npm/side-channel/-/side-channel-1.1.1.tgz",
|
||||||
@@ -1359,6 +1919,13 @@
|
|||||||
"node": ">=0.6"
|
"node": ">=0.6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tslib": {
|
||||||
|
"version": "2.8.1",
|
||||||
|
"resolved": "http://mirrors.tencentyun.com/npm/tslib/-/tslib-2.8.1.tgz",
|
||||||
|
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||||
|
"license": "0BSD",
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
"node_modules/tunnel-agent": {
|
"node_modules/tunnel-agent": {
|
||||||
"version": "0.6.0",
|
"version": "0.6.0",
|
||||||
"resolved": "http://mirrors.tencentyun.com/npm/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
"resolved": "http://mirrors.tencentyun.com/npm/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
||||||
|
|||||||
+2
-1
@@ -10,8 +10,9 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"better-sqlite3": "11.3.0",
|
"better-sqlite3": "11.3.0",
|
||||||
"express": "4.21.2",
|
"express": "4.21.2",
|
||||||
"multer": "1.4.5-lts.1",
|
|
||||||
"minimatch": "9.0.5",
|
"minimatch": "9.0.5",
|
||||||
|
"multer": "1.4.5-lts.1",
|
||||||
|
"sharp": "0.35.4",
|
||||||
"yaml": "2.6.1"
|
"yaml": "2.6.1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+35
-8
@@ -2,7 +2,7 @@
|
|||||||
// 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。
|
// 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const { buildMessages, callModel } = require('./llm');
|
const { buildMessages, callModel, encodeBlocks } = require('./llm');
|
||||||
const { executeTool, resolveInTask } = require('./tools/executor');
|
const { executeTool, resolveInTask } = require('./tools/executor');
|
||||||
const mcp = require('./mcp');
|
const mcp = require('./mcp');
|
||||||
const log = require('./log');
|
const log = require('./log');
|
||||||
@@ -185,6 +185,12 @@ function modelDisplayName(modelId) {
|
|||||||
return row ? row.display_name : modelId;
|
return row ? row.display_name : modelId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 会话绑定的模型是否支持 vision(图片回传);决定 read_file 读图行为(见 tools/executor.js)。
|
||||||
|
function modelVision(session) {
|
||||||
|
const m = session.model_id && (cfg.models || []).find(x => x.id === session.model_id);
|
||||||
|
return !!(m && m.vision);
|
||||||
|
}
|
||||||
|
|
||||||
function enqueue(sessionId) {
|
function enqueue(sessionId) {
|
||||||
const exists = db.prepare('SELECT id FROM wait_queue WHERE session_id = ?').get(sessionId);
|
const exists = db.prepare('SELECT id FROM wait_queue WHERE session_id = ?').get(sessionId);
|
||||||
if (!exists) db.prepare('INSERT INTO wait_queue (session_id, queued_at) VALUES (?,?)').run(sessionId, now());
|
if (!exists) db.prepare('INSERT INTO wait_queue (session_id, queued_at) VALUES (?,?)').run(sessionId, now());
|
||||||
@@ -419,6 +425,7 @@ async function runLoopInner(sessionId, run) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 执行工具
|
// 执行工具
|
||||||
|
const vision = modelVision(session);
|
||||||
for (const tc of msgRow.tool_calls) {
|
for (const tc of msgRow.tool_calls) {
|
||||||
if (run.cancelled) break;
|
if (run.cancelled) break;
|
||||||
const id = tc.id || ('call_' + Math.random().toString(36).slice(2, 10));
|
const id = tc.id || ('call_' + Math.random().toString(36).slice(2, 10));
|
||||||
@@ -427,16 +434,36 @@ async function runLoopInner(sessionId, run) {
|
|||||||
let output;
|
let output;
|
||||||
let isError = false;
|
let isError = false;
|
||||||
try {
|
try {
|
||||||
output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db },
|
output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db, vision },
|
||||||
tc.function.name, safeParse(tc.function.arguments));
|
tc.function.name, safeParse(tc.function.arguments));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
output = 'Error: ' + (err && err.message || String(err));
|
output = 'Error: ' + (err && err.message || String(err));
|
||||||
isError = true;
|
isError = true;
|
||||||
}
|
}
|
||||||
if (run.cancelled && !isError) output = (output ? output + '\n' : '') + '(用户已终止任务)';
|
// 工具可返回 {text, image_data_url}(read_file 读图且模型支持 vision):
|
||||||
log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + String(output || '').slice(0, 500));
|
// 落库时把图片以 image_url 内容块拼进 tool 消息(哨兵 JSON,见 llm.js encodeBlocks),
|
||||||
bus.emit(sessionId, 'tool_result', { tool_call_id: id, name: tc.function.name, content: output.slice(0, 20000), is_error: isError });
|
// buildMessages 发请求前还原为 OpenAI 内容块数组。
|
||||||
saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: output });
|
let outText;
|
||||||
|
let imageDataUrl = null;
|
||||||
|
if (output && typeof output === 'object' && output.image_data_url) {
|
||||||
|
outText = String(output.text || '');
|
||||||
|
imageDataUrl = String(output.image_data_url);
|
||||||
|
} else {
|
||||||
|
outText = String(output == null ? '' : output);
|
||||||
|
}
|
||||||
|
if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)';
|
||||||
|
log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText.slice(0, 500));
|
||||||
|
const toolContent = imageDataUrl
|
||||||
|
? encodeBlocks([
|
||||||
|
{ type: 'text', text: outText },
|
||||||
|
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
||||||
|
])
|
||||||
|
: outText;
|
||||||
|
bus.emit(sessionId, 'tool_result', {
|
||||||
|
tool_call_id: id, name: tc.function.name, content: outText.slice(0, 20000),
|
||||||
|
is_error: isError, image_url: imageDataUrl || undefined,
|
||||||
|
});
|
||||||
|
saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: toolContent });
|
||||||
}
|
}
|
||||||
iter++;
|
iter++;
|
||||||
}
|
}
|
||||||
@@ -483,10 +510,10 @@ function sessionContextContent(session) {
|
|||||||
const model = session.model_id
|
const model = session.model_id
|
||||||
? modelDisplayName(session.model_id)
|
? modelDisplayName(session.model_id)
|
||||||
: '尚未绑定(排队中,绑定后自动开始处理)';
|
: '尚未绑定(排队中,绑定后自动开始处理)';
|
||||||
// vision 能力随会话上下文告知 AI:不支持时读图只能拿到元信息,免得白跑一轮 read_file
|
// vision 能力随会话上下文告知 AI:支持时 read_file 读图会回传图片内容,不支持时只能拿到文件信息
|
||||||
const bound = session.model_id && cfg.models.find(m => m.id === session.model_id);
|
const bound = session.model_id && cfg.models.find(m => m.id === session.model_id);
|
||||||
const visionLine = bound
|
const visionLine = bound
|
||||||
? (bound.vision ? '支持' : '不支持')
|
? (bound.vision ? '支持(read_file 读图会返回压缩后的图片内容)' : '不支持(read_file 读图只能拿到文件信息)')
|
||||||
: '未知';
|
: '未知';
|
||||||
return [
|
return [
|
||||||
'【会话上下文】由系统自动注入,无需回复本消息',
|
'【会话上下文】由系统自动注入,无需回复本消息',
|
||||||
|
|||||||
+16
-3
@@ -12,6 +12,18 @@ function buildSystemPrompt() {
|
|||||||
return SYSTEM_PROMPT_TEMPLATE;
|
return SYSTEM_PROMPT_TEMPLATE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 多模态内容块存取:messages.content 列为 TEXT,含图片的 tool 消息
|
||||||
|
// 以 {"__blocks__":[content...]} 哨兵 JSON 落库(见需求 §5),
|
||||||
|
// 发请求时还原为 OpenAI 内容块数组(text + image_url)。tool/user 消息共用。
|
||||||
|
function encodeBlocks(blocks) { return JSON.stringify({ __blocks__: blocks }); }
|
||||||
|
function decodeBlocks(content) {
|
||||||
|
if (typeof content !== 'string' || !content.startsWith('{"__blocks__":')) return null;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(content);
|
||||||
|
return Array.isArray(parsed.__blocks__) ? parsed.__blocks__ : null;
|
||||||
|
} catch (_) { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
// 将 DB 消息行转为 OpenAI messages 数组(不含 system,由调用方拼接)。
|
// 将 DB 消息行转为 OpenAI messages 数组(不含 system,由调用方拼接)。
|
||||||
function buildMessages(session) {
|
function buildMessages(session) {
|
||||||
const messages = [{ role: 'system', content: SYSTEM_PROMPT_TEMPLATE }];
|
const messages = [{ role: 'system', content: SYSTEM_PROMPT_TEMPLATE }];
|
||||||
@@ -22,9 +34,10 @@ function buildMessages(session) {
|
|||||||
if (!m.content) delete m.content;
|
if (!m.content) delete m.content;
|
||||||
messages.push(m);
|
messages.push(m);
|
||||||
} else if (row.role === 'tool') {
|
} else if (row.role === 'tool') {
|
||||||
messages.push({ role: 'tool', tool_call_id: row.tool_call_id, name: row.name, content: row.content || '' });
|
// 含图片的 tool 行:content 还原为 [text, image_url] 内容块数组
|
||||||
|
messages.push({ role: 'tool', tool_call_id: row.tool_call_id, name: row.name, content: decodeBlocks(row.content) || row.content || '' });
|
||||||
} else {
|
} else {
|
||||||
messages.push({ role: row.role, content: row.content || '' });
|
messages.push({ role: row.role, content: decodeBlocks(row.content) || row.content || '' });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return messages;
|
return messages;
|
||||||
@@ -128,4 +141,4 @@ async function callModel(cfg, modelId, messages, tools, options) {
|
|||||||
return { content, tool_calls: Array.from(toolCalls.values()), finish_reason: finishReason, usage, aborted };
|
return { content, tool_calls: Array.from(toolCalls.values()), finish_reason: finishReason, usage, aborted };
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { callModel, buildMessages, buildSystemPrompt };
|
module.exports = { callModel, buildMessages, buildSystemPrompt, encodeBlocks, decodeBlocks };
|
||||||
|
|||||||
@@ -447,4 +447,10 @@ body {
|
|||||||
margin: 8px 12px 0; padding: 8px 12px; border-radius: 8px;
|
margin: 8px 12px 0; padding: 8px 12px; border-radius: 8px;
|
||||||
background: #fdecea; color: #a3352a; font-size: 12.5px;
|
background: #fdecea; color: #a3352a; font-size: 12.5px;
|
||||||
white-space: pre-wrap; word-break: break-all;
|
white-space: pre-wrap; word-break: break-all;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* read_file 读图回传的图片缩略(点开原图) */
|
||||||
|
.tool-img {
|
||||||
|
display: block; max-width: 240px; max-height: 240px;
|
||||||
|
border-radius: 6px; margin-top: 6px; cursor: zoom-in;
|
||||||
|
}
|
||||||
|
|||||||
@@ -256,13 +256,39 @@ function renderToolCard(id, name, argsStr) {
|
|||||||
return div;
|
return div;
|
||||||
}
|
}
|
||||||
|
|
||||||
function fillToolResult(id, content, isError) {
|
// tool 消息 content:纯文本,或含图片时的哨兵 JSON {"__blocks__":[...]}(read_file 读图回传,见 llm.js)
|
||||||
|
function parseToolContent(content) {
|
||||||
|
if (typeof content === 'string' && content.startsWith('{"__blocks__":')) {
|
||||||
|
try {
|
||||||
|
const blocks = JSON.parse(content).__blocks__ || [];
|
||||||
|
const img = blocks.find(b => b.type === 'image_url');
|
||||||
|
return {
|
||||||
|
text: blocks.filter(b => b.type === 'text').map(b => b.text).join('\n'),
|
||||||
|
imageUrl: img && img.image_url && img.image_url.url || null,
|
||||||
|
};
|
||||||
|
} catch (_) { /* 解析失败按原文显示 */ }
|
||||||
|
}
|
||||||
|
return { text: content, imageUrl: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
function fillToolResult(id, content, isError, liveImageUrl) {
|
||||||
const card = document.getElementById('tool-' + id);
|
const card = document.getElementById('tool-' + id);
|
||||||
if (!card) return;
|
if (!card) return;
|
||||||
if (isError) card.classList.add('error');
|
if (isError) card.classList.add('error');
|
||||||
card.querySelector('.spin').textContent = isError ? '✕' : '✓';
|
card.querySelector('.spin').textContent = isError ? '✕' : '✓';
|
||||||
|
const parsed = parseToolContent(content);
|
||||||
const pre = card.querySelector('.tool-result');
|
const pre = card.querySelector('.tool-result');
|
||||||
if (pre) pre.textContent = content || '(空)';
|
if (!pre) return;
|
||||||
|
pre.textContent = parsed.text || '(空)';
|
||||||
|
const img = liveImageUrl || parsed.imageUrl;
|
||||||
|
if (img && !card.querySelector('.tool-img')) {
|
||||||
|
const el = document.createElement('img');
|
||||||
|
el.className = 'tool-img';
|
||||||
|
el.src = img;
|
||||||
|
el.title = '点击查看详情';
|
||||||
|
el.onclick = () => window.open(el.src);
|
||||||
|
pre.insertAdjacentElement('afterend', el);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function msgNode(m, cardMap) {
|
function msgNode(m, cardMap) {
|
||||||
@@ -421,7 +447,7 @@ function connectStream() {
|
|||||||
});
|
});
|
||||||
es.addEventListener('tool_result', ev => {
|
es.addEventListener('tool_result', ev => {
|
||||||
const d = JSON.parse(ev.data);
|
const d = JSON.parse(ev.data);
|
||||||
fillToolResult(d.tool_call_id, d.content, d.is_error);
|
fillToolResult(d.tool_call_id, d.content, d.is_error, d.image_url);
|
||||||
scrollBottom();
|
scrollBottom();
|
||||||
});
|
});
|
||||||
es.addEventListener('done', ev => {
|
es.addEventListener('done', ev => {
|
||||||
|
|||||||
@@ -85,6 +85,12 @@ function truncate(text, max) {
|
|||||||
const AUTO_OUTLINE_SIZE = 16384;
|
const AUTO_OUTLINE_SIZE = 16384;
|
||||||
const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.tiff', '.tif']);
|
const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.tiff', '.tif']);
|
||||||
|
|
||||||
|
// read_file 读图(仅当会话绑定的模型 vision: true):sharp 压缩后转 base64 data URL,
|
||||||
|
// 由 engine 拼成下一轮请求里 tool 消息的 image_url 内容块。
|
||||||
|
const IMG_MAX_INPUT_BYTES = 50 * 1024 * 1024; // 与上传上限一致
|
||||||
|
const IMG_MAX_SIDE = 2000; // 长边压缩阈值(px)
|
||||||
|
const IMG_JPEG_QUALITY = 80;
|
||||||
|
|
||||||
// 简化版符号大纲(Zed 用 tree-sitter,这里用正则提取常见语言的顶层符号 + Markdown 标题)。
|
// 简化版符号大纲(Zed 用 tree-sitter,这里用正则提取常见语言的顶层符号 + Markdown 标题)。
|
||||||
// 返回 "# File outline ..." 文本;无法提取时返回 null(调用方回退到首 1KB)。
|
// 返回 "# File outline ..." 文本;无法提取时返回 null(调用方回退到首 1KB)。
|
||||||
const OUTLINE_RES = [
|
const OUTLINE_RES = [
|
||||||
@@ -130,6 +136,29 @@ function outlineWrapper(outlineText) {
|
|||||||
'For example, to read a function shown as [L100-150], use start_line: 100 and end_line: 150.';
|
'For example, to read a function shown as [L100-150], use start_line: 100 and end_line: 150.';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 用 sharp 压缩(按 EXIF 摆正、长边 ≤2000px、JPEG 质量 80)并转 base64 data URL。
|
||||||
|
// 成功返回 {text, image_data_url},engine 侧拼成 tool 消息的多模态内容块;
|
||||||
|
// sharp 缺失或解码失败时退回纯文本提示,不让整个工具回合报错。
|
||||||
|
async function readImageAsBlocks(abs, rel, size) {
|
||||||
|
let sharp;
|
||||||
|
try { sharp = require('sharp'); }
|
||||||
|
catch (_) { return '[图片文件] ' + rel + '(' + size + ' 字节)。服务端未安装 sharp,无法压缩回传图片内容。'; }
|
||||||
|
try {
|
||||||
|
const r = await sharp(abs)
|
||||||
|
.rotate()
|
||||||
|
.resize({ width: IMG_MAX_SIDE, height: IMG_MAX_SIDE, fit: 'inside', withoutEnlargement: true })
|
||||||
|
.jpeg({ quality: IMG_JPEG_QUALITY })
|
||||||
|
.toBuffer({ resolveWithObject: true });
|
||||||
|
return {
|
||||||
|
text: '[图片文件] ' + rel + '(原 ' + size + ' 字节,已压缩为 ' + r.info.width + '×' + r.info.height +
|
||||||
|
' JPEG ' + r.data.length + ' 字节,图片见同条消息的 image_url 内容块)',
|
||||||
|
image_data_url: 'data:image/jpeg;base64,' + r.data.toString('base64'),
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return '[图片文件] ' + rel + '(' + size + ' 字节),图片解码/压缩失败:' + (err && err.message || err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function toolReadFile(ctx, args) {
|
async function toolReadFile(ctx, args) {
|
||||||
const abs = resolveInTask(ctx.taskDir, args.path);
|
const abs = resolveInTask(ctx.taskDir, args.path);
|
||||||
let st;
|
let st;
|
||||||
@@ -137,7 +166,14 @@ async function toolReadFile(ctx, args) {
|
|||||||
catch (_) { throw new Error('文件不存在:' + args.path); }
|
catch (_) { throw new Error('文件不存在:' + args.path); }
|
||||||
if (st.isDirectory()) throw new Error(args.path + ' 是目录不是文件,请用 list_directory 工具查看目录内容。');
|
if (st.isDirectory()) throw new Error(args.path + ' 是目录不是文件,请用 list_directory 工具查看目录内容。');
|
||||||
if (IMAGE_EXTS.has(path.extname(abs).toLowerCase())) {
|
if (IMAGE_EXTS.has(path.extname(abs).toLowerCase())) {
|
||||||
return '[图片文件] ' + relOf(ctx.taskDir, abs) + '(' + st.size + ' 字节)。当前服务无法把图片内容回传给模型。';
|
const rel = relOf(ctx.taskDir, abs);
|
||||||
|
if (!ctx.vision) {
|
||||||
|
return '[图片文件] ' + rel + '(' + st.size + ' 字节)。图片文件,当前模型不支持视觉识别,无法回传图片内容。';
|
||||||
|
}
|
||||||
|
if (st.size > IMG_MAX_INPUT_BYTES) {
|
||||||
|
return '[图片文件] ' + rel + '(' + st.size + ' 字节)过大(上限 ' + IMG_MAX_INPUT_BYTES + ' 字节),已跳过回传。';
|
||||||
|
}
|
||||||
|
return await readImageAsBlocks(abs, rel, st.size);
|
||||||
}
|
}
|
||||||
if (isProbablyBinary(abs)) return '[二进制文件,跳过文本读取] ' + relOf(ctx.taskDir, abs);
|
if (isProbablyBinary(abs)) return '[二进制文件,跳过文本读取] ' + relOf(ctx.taskDir, abs);
|
||||||
const text = await fsp.readFile(abs, 'utf8');
|
const text = await fsp.readFile(abs, 'utf8');
|
||||||
@@ -646,7 +682,8 @@ const HANDLERS = {
|
|||||||
run_skill: (ctx, args) => skills.runSkill(ctx, args.skill_name, args.input_args),
|
run_skill: (ctx, args) => skills.runSkill(ctx, args.skill_name, args.input_args),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ctx: {sessionId, taskDir, run, cfg, db}
|
// ctx: {sessionId, taskDir, run, cfg, db, vision}
|
||||||
|
// 返回值一般为字符串;read_file 读图且 ctx.vision 为 true 时返回 {text, image_data_url}。
|
||||||
// 内置工具优先;未命中时交给 MCP 注册表(启动时发现的服务工具)。
|
// 内置工具优先;未命中时交给 MCP 注册表(启动时发现的服务工具)。
|
||||||
async function executeTool(baseCtx, name, args) {
|
async function executeTool(baseCtx, name, args) {
|
||||||
const handler = HANDLERS[name];
|
const handler = HANDLERS[name];
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
# 冒烟测试用配置:上游指向本地 mock
|
# 冒烟测试用配置:上游指向本地 mock
|
||||||
|
# 两个模型都开 vision:让「读图」端到端测试与绑定哪个模型无关(非 vision 分支由单测覆盖)
|
||||||
models:
|
models:
|
||||||
- id: ds1
|
- id: ds1
|
||||||
display_name: "Mock Model 1"
|
display_name: "Mock Model 1"
|
||||||
url: "http://127.0.0.1:9999/v1"
|
url: "http://127.0.0.1:9999/v1"
|
||||||
|
vision: true
|
||||||
- id: ds2
|
- id: ds2
|
||||||
display_name: "Mock Model 2"
|
display_name: "Mock Model 2"
|
||||||
url: "http://127.0.0.1:9999/v1"
|
url: "http://127.0.0.1:9999/v1"
|
||||||
|
vision: true
|
||||||
|
|
||||||
workspace_root: "./test/tmp/tasks"
|
workspace_root: "./test/tmp/tasks"
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
// 规则:
|
// 规则:
|
||||||
// - 历史里已有 tool 结果 -> 回纯文本收尾
|
// - 历史里已有 tool 结果 -> 回纯文本收尾
|
||||||
// - 最后一条 user 消息含 "写文件" -> 返回 write_file 工具调用
|
// - 最后一条 user 消息含 "写文件" -> 返回 write_file 工具调用
|
||||||
|
// - 含 "读图" -> 返回 read_file(pic.png) 工具调用(验证 vision 图片回传)
|
||||||
// - 含 "慢" -> 慢速输出(用于终止测试)
|
// - 含 "慢" -> 慢速输出(用于终止测试)
|
||||||
// - 其他 -> 纯文本 "已收到: <最后一条user消息>"
|
// - 其他 -> 纯文本 "已收到: <最后一条user消息>"
|
||||||
const http = require('http');
|
const http = require('http');
|
||||||
@@ -40,6 +41,17 @@ function streamToolCall(res) {
|
|||||||
res.end();
|
res.end();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function streamReadFileCall(res) {
|
||||||
|
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
||||||
|
const id = 'call-img-' + Date.now();
|
||||||
|
sse(res, { choices: [{ delta: { role: 'assistant', tool_calls: [{ id, function: { name: 'read_file' } }] } }] });
|
||||||
|
sse(res, { choices: [{ delta: { tool_calls: [{ function: { arguments: '{"path": "pic.png"}' } }] } }] });
|
||||||
|
sse(res, { choices: [{ delta: {}, finish_reason: 'tool_calls' }] });
|
||||||
|
sse(res, { usage: { prompt_tokens: 60, completion_tokens: 10, total_tokens: 70 } });
|
||||||
|
res.write('data: [DONE]\n\n');
|
||||||
|
res.end();
|
||||||
|
}
|
||||||
|
|
||||||
const server = http.createServer((req, res) => {
|
const server = http.createServer((req, res) => {
|
||||||
if (!req.url.includes('/chat/completions')) {
|
if (!req.url.includes('/chat/completions')) {
|
||||||
res.writeHead(404).end('not found');
|
res.writeHead(404).end('not found');
|
||||||
@@ -60,6 +72,8 @@ const server = http.createServer((req, res) => {
|
|||||||
process.stdout = process.stdout;
|
process.stdout = process.stdout;
|
||||||
if (hasToolResult) {
|
if (hasToolResult) {
|
||||||
streamText(res, '文件已创建并验证完毕。');
|
streamText(res, '文件已创建并验证完毕。');
|
||||||
|
} else if (userText.includes('读图')) {
|
||||||
|
streamReadFileCall(res);
|
||||||
} else if (userText.includes('写文件')) {
|
} else if (userText.includes('写文件')) {
|
||||||
streamToolCall(res);
|
streamToolCall(res);
|
||||||
} else if (userText.includes('慢')) {
|
} else if (userText.includes('慢')) {
|
||||||
|
|||||||
@@ -118,6 +118,26 @@ async function main() {
|
|||||||
ok(!!taskMd, '开始执行自动追加模板消息');
|
ok(!!taskMd, '开始执行自动追加模板消息');
|
||||||
ok(d.session.phase === 'execution', '阶段切换为 execution');
|
ok(d.session.phase === 'execution', '阶段切换为 execution');
|
||||||
|
|
||||||
|
// ---- 2.5 read_file 读图回传(vision)----
|
||||||
|
console.log('\n[2.5] 读图回传');
|
||||||
|
const sharp = require('sharp');
|
||||||
|
const pngBuf = await sharp({ create: { width: 8, height: 8, channels: 3, background: { r: 200, g: 30, b: 30 } } }).png().toBuffer();
|
||||||
|
const si = await api('POST', '/api/sessions', { title: '冒烟-读图' });
|
||||||
|
await waitSession(si.id, x => x.session.status === 'active', 5000, '绑定');
|
||||||
|
const fdImg = new FormData();
|
||||||
|
fdImg.append('files', new Blob([pngBuf], { type: 'image/png' }), 'pic.png');
|
||||||
|
const upImg = await fetch(BASE + '/api/sessions/' + si.id + '/files', { method: 'POST', body: fdImg });
|
||||||
|
ok(upImg.ok, '上传图片到任务目录');
|
||||||
|
await api('POST', '/api/sessions/' + si.id + '/messages', { content: '读图 pic.png' });
|
||||||
|
d = await waitSession(si.id, x => x.session.status === 'held' && x.messages.some(m => m.role === 'tool'), 15000, '读图回合完成');
|
||||||
|
const imgTool = d.messages.filter(m => m.role === 'tool')
|
||||||
|
.map(m => { try { return JSON.parse(m.content); } catch (_) { return null; } })
|
||||||
|
.find(j => j && Array.isArray(j.__blocks__));
|
||||||
|
ok(!!imgTool && imgTool.__blocks__.some(b =>
|
||||||
|
b.type === 'image_url' && String(b.image_url && b.image_url.url).startsWith('data:image/jpeg;base64,')),
|
||||||
|
'tool 消息含压缩后的 image_url 内容块',
|
||||||
|
imgTool ? JSON.stringify(imgTool.__blocks__.map(b => b.type)) : '无多模态块');
|
||||||
|
|
||||||
// ---- 3. 路径穿越防护 ----
|
// ---- 3. 路径穿越防护 ----
|
||||||
console.log('\n[3] 路径防护');
|
console.log('\n[3] 路径防护');
|
||||||
let blocked = false;
|
let blocked = false;
|
||||||
@@ -197,6 +217,15 @@ async function main() {
|
|||||||
ok(fpath.includes('g.txt'), 'find_path 命中', fpath);
|
ok(fpath.includes('g.txt'), 'find_path 命中', fpath);
|
||||||
const esc = await executeTool(ctx, 'read_file', { path: '../outside.txt' }).catch(e => 'ERR:' + e.message);
|
const esc = await executeTool(ctx, 'read_file', { path: '../outside.txt' }).catch(e => 'ERR:' + e.message);
|
||||||
ok(String(esc).includes('越界'), 'read_file 拒绝越界', String(esc));
|
ok(String(esc).includes('越界'), 'read_file 拒绝越界', String(esc));
|
||||||
|
// read_file 读图:非 vision → 文本提示;vision → {text, image_data_url}(依赖 sharp)
|
||||||
|
const unitPng = await require('sharp')({ create: { width: 4, height: 4, channels: 3, background: { r: 10, g: 120, b: 200 } } }).png().toBuffer();
|
||||||
|
fs.writeFileSync(path.join(ctx.taskDir, 'pic.png'), unitPng);
|
||||||
|
const imgNoVision = await executeTool(ctx, 'read_file', { path: 'pic.png' });
|
||||||
|
ok(typeof imgNoVision === 'string' && imgNoVision.includes('不支持视觉识别'), 'read_file 非 vision 图片返回文本提示', String(imgNoVision).slice(0, 80));
|
||||||
|
const ctxVision = Object.assign({}, ctx, { vision: true });
|
||||||
|
const imgVision = await executeTool(ctxVision, 'read_file', { path: 'pic.png' });
|
||||||
|
ok(imgVision && typeof imgVision === 'object' && String(imgVision.image_data_url).startsWith('data:image/jpeg;base64,'),
|
||||||
|
'read_file vision 图片返回压缩 data URL', imgVision && imgVision.text ? String(imgVision.text).slice(0, 90) : String(imgVision).slice(0, 90));
|
||||||
|
|
||||||
// ---- 8. 配置读写 API(阶段 2)----
|
// ---- 8. 配置读写 API(阶段 2)----
|
||||||
console.log('\n[8] 配置读写');
|
console.log('\n[8] 配置读写');
|
||||||
|
|||||||
Reference in New Issue
Block a user