From f895801ad2772a32c5c65a2f0b0ec3c3f27f3b5c Mon Sep 17 00:00:00 2001 From: kicer Date: Wed, 2 Sep 2026 10:52:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=A4=9A=E6=A8=A1=E6=80=81?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E8=AF=86=E5=88=AB=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + app/config.example.yml | 2 +- app/package-lock.json | 571 +++++++++++++++++++++++++++++++- app/package.json | 3 +- app/server/engine.js | 43 ++- app/server/llm.js | 19 +- app/server/public/css/style.css | 10 +- app/server/public/js/tasks.js | 32 +- app/server/tools/executor.js | 41 ++- app/test/config.test.yml | 3 + app/test/mock-upstream.js | 14 + app/test/smoke.js | 29 ++ 12 files changed, 746 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6a4d8f6..0df0480 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - 任务状态开关(规划 / 执行) - 排队机制,用户可离线输入 - 工具调用(文件操作、终端、网络抓取、网页搜索) +- 多模态图片识别(模型配置 vision: true 时,read_file 读图会压缩回传图片内容;用户上传仅存任务目录,不自动发送) - 借鉴 Zed agent 的 system prompt 和工具定义 --- diff --git a/app/config.example.yml b/app/config.example.yml index 60331f9..9085c78 100644 --- a/app/config.example.yml +++ b/app/config.example.yml @@ -8,7 +8,7 @@ models: display_name: "AMD DeepSeek 1" url: "https://proxy-a.example.com/v1" # 独立的 OpenAI 兼容 /v1 地址 api_key: "sk-..." # 各模型可不同,留空则不带鉴权头 - # vision: true # 选填,默认 false;支持图片时会话上下文会告知 AI + # vision: true # 选填,默认 false;支持时 read_file 读图会压缩回传(长边≤2000px JPEG),会话上下文会告知 AI - id: ds2 display_name: "AMD DeepSeek 2" url: "https://proxy-b.example.com/v1" diff --git a/app/package-lock.json b/app/package-lock.json index f787c0c..0c8d880 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,20 +1,531 @@ { "name": "agent-app", - "version": "1.0.0", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-app", - "version": "1.0.0", + "version": "2.0.0", "dependencies": { "better-sqlite3": "11.3.0", "express": "4.21.2", "minimatch": "9.0.5", "multer": "1.4.5-lts.1", + "sharp": "0.35.4", "yaml": "2.6.1" } }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "http://mirrors.tencentyun.com/npm/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "http://mirrors.tencentyun.com/npm/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", + "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", + "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", + "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", + "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", + "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", + "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", + "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", + "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", + "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", + "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", + "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", + "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.3", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", + "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", + "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", + "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", + "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", + "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", + "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", + "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", + "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", + "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", + "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", + "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", + "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", + "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", + "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "http://mirrors.tencentyun.com/npm/accepts/-/accepts-1.3.8.tgz", @@ -1151,6 +1662,55 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.35.4", + "resolved": "http://mirrors.tencentyun.com/npm/sharp/-/sharp-0.35.4.tgz", + "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.4", + "@img/sharp-darwin-x64": "0.35.4", + "@img/sharp-freebsd-wasm32": "0.35.4", + "@img/sharp-libvips-darwin-arm64": "1.3.3", + "@img/sharp-libvips-darwin-x64": "1.3.3", + "@img/sharp-libvips-linux-arm": "1.3.3", + "@img/sharp-libvips-linux-arm64": "1.3.3", + "@img/sharp-libvips-linux-ppc64": "1.3.3", + "@img/sharp-libvips-linux-riscv64": "1.3.3", + "@img/sharp-libvips-linux-s390x": "1.3.3", + "@img/sharp-libvips-linux-x64": "1.3.3", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", + "@img/sharp-libvips-linuxmusl-x64": "1.3.3", + "@img/sharp-linux-arm": "0.35.4", + "@img/sharp-linux-arm64": "0.35.4", + "@img/sharp-linux-ppc64": "0.35.4", + "@img/sharp-linux-riscv64": "0.35.4", + "@img/sharp-linux-s390x": "0.35.4", + "@img/sharp-linux-x64": "0.35.4", + "@img/sharp-linuxmusl-arm64": "0.35.4", + "@img/sharp-linuxmusl-x64": "0.35.4", + "@img/sharp-webcontainers-wasm32": "0.35.4", + "@img/sharp-win32-arm64": "0.35.4", + "@img/sharp-win32-ia32": "0.35.4", + "@img/sharp-win32-x64": "0.35.4" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, "node_modules/side-channel": { "version": "1.1.1", "resolved": "http://mirrors.tencentyun.com/npm/side-channel/-/side-channel-1.1.1.tgz", @@ -1359,6 +1919,13 @@ "node": ">=0.6" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "http://mirrors.tencentyun.com/npm/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "http://mirrors.tencentyun.com/npm/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/app/package.json b/app/package.json index 0c43d73..e0d2e82 100644 --- a/app/package.json +++ b/app/package.json @@ -10,8 +10,9 @@ "dependencies": { "better-sqlite3": "11.3.0", "express": "4.21.2", - "multer": "1.4.5-lts.1", "minimatch": "9.0.5", + "multer": "1.4.5-lts.1", + "sharp": "0.35.4", "yaml": "2.6.1" } } diff --git a/app/server/engine.js b/app/server/engine.js index e47a30a..e2e2550 100644 --- a/app/server/engine.js +++ b/app/server/engine.js @@ -2,7 +2,7 @@ // 单进程内存模型:JS 单线程 + better-sqlite3 同步 API,不存在跨进程竞争。 const fs = require('fs'); const path = require('path'); -const { buildMessages, callModel } = require('./llm'); +const { buildMessages, callModel, encodeBlocks } = require('./llm'); const { executeTool, resolveInTask } = require('./tools/executor'); const mcp = require('./mcp'); const log = require('./log'); @@ -185,6 +185,12 @@ function modelDisplayName(modelId) { return row ? row.display_name : modelId; } +// 会话绑定的模型是否支持 vision(图片回传);决定 read_file 读图行为(见 tools/executor.js)。 +function modelVision(session) { + const m = session.model_id && (cfg.models || []).find(x => x.id === session.model_id); + return !!(m && m.vision); +} + function enqueue(sessionId) { const exists = db.prepare('SELECT id FROM wait_queue WHERE session_id = ?').get(sessionId); if (!exists) db.prepare('INSERT INTO wait_queue (session_id, queued_at) VALUES (?,?)').run(sessionId, now()); @@ -419,6 +425,7 @@ async function runLoopInner(sessionId, run) { } // 执行工具 + const vision = modelVision(session); for (const tc of msgRow.tool_calls) { if (run.cancelled) break; const id = tc.id || ('call_' + Math.random().toString(36).slice(2, 10)); @@ -427,16 +434,36 @@ async function runLoopInner(sessionId, run) { let output; let isError = false; try { - output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db }, + output = await executeTool({ sessionId, taskDir: taskDirOf(sessionId), run, cfg, db, vision }, tc.function.name, safeParse(tc.function.arguments)); } catch (err) { output = 'Error: ' + (err && err.message || String(err)); isError = true; } - if (run.cancelled && !isError) output = (output ? output + '\n' : '') + '(用户已终止任务)'; - log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + String(output || '').slice(0, 500)); - bus.emit(sessionId, 'tool_result', { tool_call_id: id, name: tc.function.name, content: output.slice(0, 20000), is_error: isError }); - saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: output }); + // 工具可返回 {text, image_data_url}(read_file 读图且模型支持 vision): + // 落库时把图片以 image_url 内容块拼进 tool 消息(哨兵 JSON,见 llm.js encodeBlocks), + // buildMessages 发请求前还原为 OpenAI 内容块数组。 + let outText; + let imageDataUrl = null; + if (output && typeof output === 'object' && output.image_data_url) { + outText = String(output.text || ''); + imageDataUrl = String(output.image_data_url); + } else { + outText = String(output == null ? '' : output); + } + if (run.cancelled && !isError) outText = (outText ? outText + '\n' : '') + '(用户已终止任务)'; + log.debug('[tool] ' + sessionId + ' ' + tc.function.name + (isError ? ' 失败: ' : ' 完成: ') + outText.slice(0, 500)); + const toolContent = imageDataUrl + ? encodeBlocks([ + { type: 'text', text: outText }, + { type: 'image_url', image_url: { url: imageDataUrl } }, + ]) + : outText; + bus.emit(sessionId, 'tool_result', { + tool_call_id: id, name: tc.function.name, content: outText.slice(0, 20000), + is_error: isError, image_url: imageDataUrl || undefined, + }); + saveMessage(sessionId, { role: 'tool', tool_call_id: id, name: tc.function.name, content: toolContent }); } iter++; } @@ -483,10 +510,10 @@ function sessionContextContent(session) { const model = session.model_id ? modelDisplayName(session.model_id) : '尚未绑定(排队中,绑定后自动开始处理)'; - // vision 能力随会话上下文告知 AI:不支持时读图只能拿到元信息,免得白跑一轮 read_file + // vision 能力随会话上下文告知 AI:支持时 read_file 读图会回传图片内容,不支持时只能拿到文件信息 const bound = session.model_id && cfg.models.find(m => m.id === session.model_id); const visionLine = bound - ? (bound.vision ? '支持' : '不支持') + ? (bound.vision ? '支持(read_file 读图会返回压缩后的图片内容)' : '不支持(read_file 读图只能拿到文件信息)') : '未知'; return [ '【会话上下文】由系统自动注入,无需回复本消息', diff --git a/app/server/llm.js b/app/server/llm.js index cc3b62b..59628f3 100644 --- a/app/server/llm.js +++ b/app/server/llm.js @@ -12,6 +12,18 @@ function buildSystemPrompt() { return SYSTEM_PROMPT_TEMPLATE; } +// 多模态内容块存取:messages.content 列为 TEXT,含图片的 tool 消息 +// 以 {"__blocks__":[content...]} 哨兵 JSON 落库(见需求 §5), +// 发请求时还原为 OpenAI 内容块数组(text + image_url)。tool/user 消息共用。 +function encodeBlocks(blocks) { return JSON.stringify({ __blocks__: blocks }); } +function decodeBlocks(content) { + if (typeof content !== 'string' || !content.startsWith('{"__blocks__":')) return null; + try { + const parsed = JSON.parse(content); + return Array.isArray(parsed.__blocks__) ? parsed.__blocks__ : null; + } catch (_) { return null; } +} + // 将 DB 消息行转为 OpenAI messages 数组(不含 system,由调用方拼接)。 function buildMessages(session) { const messages = [{ role: 'system', content: SYSTEM_PROMPT_TEMPLATE }]; @@ -22,9 +34,10 @@ function buildMessages(session) { if (!m.content) delete m.content; messages.push(m); } else if (row.role === 'tool') { - messages.push({ role: 'tool', tool_call_id: row.tool_call_id, name: row.name, content: row.content || '' }); + // 含图片的 tool 行:content 还原为 [text, image_url] 内容块数组 + messages.push({ role: 'tool', tool_call_id: row.tool_call_id, name: row.name, content: decodeBlocks(row.content) || row.content || '' }); } else { - messages.push({ role: row.role, content: row.content || '' }); + messages.push({ role: row.role, content: decodeBlocks(row.content) || row.content || '' }); } } return messages; @@ -128,4 +141,4 @@ async function callModel(cfg, modelId, messages, tools, options) { return { content, tool_calls: Array.from(toolCalls.values()), finish_reason: finishReason, usage, aborted }; } -module.exports = { callModel, buildMessages, buildSystemPrompt }; +module.exports = { callModel, buildMessages, buildSystemPrompt, encodeBlocks, decodeBlocks }; diff --git a/app/server/public/css/style.css b/app/server/public/css/style.css index 9e890db..f419a23 100644 --- a/app/server/public/css/style.css +++ b/app/server/public/css/style.css @@ -446,5 +446,11 @@ body { .cfg-err { margin: 8px 12px 0; padding: 8px 12px; border-radius: 8px; background: #fdecea; color: #a3352a; font-size: 12.5px; - white-space: pre-wrap; word-break: break-all; -} + white-space: pre-wrap; word-break: break-all; + } + + /* read_file 读图回传的图片缩略(点开原图) */ + .tool-img { + display: block; max-width: 240px; max-height: 240px; + border-radius: 6px; margin-top: 6px; cursor: zoom-in; + } diff --git a/app/server/public/js/tasks.js b/app/server/public/js/tasks.js index 40d3f2a..aa70436 100644 --- a/app/server/public/js/tasks.js +++ b/app/server/public/js/tasks.js @@ -256,13 +256,39 @@ function renderToolCard(id, name, argsStr) { return div; } -function fillToolResult(id, content, isError) { +// tool 消息 content:纯文本,或含图片时的哨兵 JSON {"__blocks__":[...]}(read_file 读图回传,见 llm.js) +function parseToolContent(content) { + if (typeof content === 'string' && content.startsWith('{"__blocks__":')) { + try { + const blocks = JSON.parse(content).__blocks__ || []; + const img = blocks.find(b => b.type === 'image_url'); + return { + text: blocks.filter(b => b.type === 'text').map(b => b.text).join('\n'), + imageUrl: img && img.image_url && img.image_url.url || null, + }; + } catch (_) { /* 解析失败按原文显示 */ } + } + return { text: content, imageUrl: null }; +} + +function fillToolResult(id, content, isError, liveImageUrl) { const card = document.getElementById('tool-' + id); if (!card) return; if (isError) card.classList.add('error'); card.querySelector('.spin').textContent = isError ? '✕' : '✓'; + const parsed = parseToolContent(content); const pre = card.querySelector('.tool-result'); - if (pre) pre.textContent = content || '(空)'; + if (!pre) return; + pre.textContent = parsed.text || '(空)'; + const img = liveImageUrl || parsed.imageUrl; + if (img && !card.querySelector('.tool-img')) { + const el = document.createElement('img'); + el.className = 'tool-img'; + el.src = img; + el.title = '点击查看详情'; + el.onclick = () => window.open(el.src); + pre.insertAdjacentElement('afterend', el); + } } function msgNode(m, cardMap) { @@ -421,7 +447,7 @@ function connectStream() { }); es.addEventListener('tool_result', ev => { const d = JSON.parse(ev.data); - fillToolResult(d.tool_call_id, d.content, d.is_error); + fillToolResult(d.tool_call_id, d.content, d.is_error, d.image_url); scrollBottom(); }); es.addEventListener('done', ev => { diff --git a/app/server/tools/executor.js b/app/server/tools/executor.js index ead4dc2..e6c62a4 100644 --- a/app/server/tools/executor.js +++ b/app/server/tools/executor.js @@ -85,6 +85,12 @@ function truncate(text, max) { const AUTO_OUTLINE_SIZE = 16384; const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.tiff', '.tif']); +// read_file 读图(仅当会话绑定的模型 vision: true):sharp 压缩后转 base64 data URL, +// 由 engine 拼成下一轮请求里 tool 消息的 image_url 内容块。 +const IMG_MAX_INPUT_BYTES = 50 * 1024 * 1024; // 与上传上限一致 +const IMG_MAX_SIDE = 2000; // 长边压缩阈值(px) +const IMG_JPEG_QUALITY = 80; + // 简化版符号大纲(Zed 用 tree-sitter,这里用正则提取常见语言的顶层符号 + Markdown 标题)。 // 返回 "# File outline ..." 文本;无法提取时返回 null(调用方回退到首 1KB)。 const OUTLINE_RES = [ @@ -130,6 +136,29 @@ function outlineWrapper(outlineText) { 'For example, to read a function shown as [L100-150], use start_line: 100 and end_line: 150.'; } +// 用 sharp 压缩(按 EXIF 摆正、长边 ≤2000px、JPEG 质量 80)并转 base64 data URL。 +// 成功返回 {text, image_data_url},engine 侧拼成 tool 消息的多模态内容块; +// sharp 缺失或解码失败时退回纯文本提示,不让整个工具回合报错。 +async function readImageAsBlocks(abs, rel, size) { + let sharp; + try { sharp = require('sharp'); } + catch (_) { return '[图片文件] ' + rel + '(' + size + ' 字节)。服务端未安装 sharp,无法压缩回传图片内容。'; } + try { + const r = await sharp(abs) + .rotate() + .resize({ width: IMG_MAX_SIDE, height: IMG_MAX_SIDE, fit: 'inside', withoutEnlargement: true }) + .jpeg({ quality: IMG_JPEG_QUALITY }) + .toBuffer({ resolveWithObject: true }); + return { + text: '[图片文件] ' + rel + '(原 ' + size + ' 字节,已压缩为 ' + r.info.width + '×' + r.info.height + + ' JPEG ' + r.data.length + ' 字节,图片见同条消息的 image_url 内容块)', + image_data_url: 'data:image/jpeg;base64,' + r.data.toString('base64'), + }; + } catch (err) { + return '[图片文件] ' + rel + '(' + size + ' 字节),图片解码/压缩失败:' + (err && err.message || err); + } +} + async function toolReadFile(ctx, args) { const abs = resolveInTask(ctx.taskDir, args.path); let st; @@ -137,7 +166,14 @@ async function toolReadFile(ctx, args) { catch (_) { throw new Error('文件不存在:' + args.path); } if (st.isDirectory()) throw new Error(args.path + ' 是目录不是文件,请用 list_directory 工具查看目录内容。'); if (IMAGE_EXTS.has(path.extname(abs).toLowerCase())) { - return '[图片文件] ' + relOf(ctx.taskDir, abs) + '(' + st.size + ' 字节)。当前服务无法把图片内容回传给模型。'; + const rel = relOf(ctx.taskDir, abs); + if (!ctx.vision) { + return '[图片文件] ' + rel + '(' + st.size + ' 字节)。图片文件,当前模型不支持视觉识别,无法回传图片内容。'; + } + if (st.size > IMG_MAX_INPUT_BYTES) { + return '[图片文件] ' + rel + '(' + st.size + ' 字节)过大(上限 ' + IMG_MAX_INPUT_BYTES + ' 字节),已跳过回传。'; + } + return await readImageAsBlocks(abs, rel, st.size); } if (isProbablyBinary(abs)) return '[二进制文件,跳过文本读取] ' + relOf(ctx.taskDir, abs); const text = await fsp.readFile(abs, 'utf8'); @@ -646,7 +682,8 @@ const HANDLERS = { run_skill: (ctx, args) => skills.runSkill(ctx, args.skill_name, args.input_args), }; -// ctx: {sessionId, taskDir, run, cfg, db} +// ctx: {sessionId, taskDir, run, cfg, db, vision} +// 返回值一般为字符串;read_file 读图且 ctx.vision 为 true 时返回 {text, image_data_url}。 // 内置工具优先;未命中时交给 MCP 注册表(启动时发现的服务工具)。 async function executeTool(baseCtx, name, args) { const handler = HANDLERS[name]; diff --git a/app/test/config.test.yml b/app/test/config.test.yml index ff26a6a..ee30d71 100644 --- a/app/test/config.test.yml +++ b/app/test/config.test.yml @@ -1,11 +1,14 @@ # 冒烟测试用配置:上游指向本地 mock +# 两个模型都开 vision:让「读图」端到端测试与绑定哪个模型无关(非 vision 分支由单测覆盖) models: - id: ds1 display_name: "Mock Model 1" url: "http://127.0.0.1:9999/v1" + vision: true - id: ds2 display_name: "Mock Model 2" url: "http://127.0.0.1:9999/v1" + vision: true workspace_root: "./test/tmp/tasks" diff --git a/app/test/mock-upstream.js b/app/test/mock-upstream.js index 0e1ba3d..785c31c 100644 --- a/app/test/mock-upstream.js +++ b/app/test/mock-upstream.js @@ -2,6 +2,7 @@ // 规则: // - 历史里已有 tool 结果 -> 回纯文本收尾 // - 最后一条 user 消息含 "写文件" -> 返回 write_file 工具调用 +// - 含 "读图" -> 返回 read_file(pic.png) 工具调用(验证 vision 图片回传) // - 含 "慢" -> 慢速输出(用于终止测试) // - 其他 -> 纯文本 "已收到: <最后一条user消息>" const http = require('http'); @@ -40,6 +41,17 @@ function streamToolCall(res) { res.end(); } +function streamReadFileCall(res) { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + const id = 'call-img-' + Date.now(); + sse(res, { choices: [{ delta: { role: 'assistant', tool_calls: [{ id, function: { name: 'read_file' } }] } }] }); + sse(res, { choices: [{ delta: { tool_calls: [{ function: { arguments: '{"path": "pic.png"}' } }] } }] }); + sse(res, { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }); + sse(res, { usage: { prompt_tokens: 60, completion_tokens: 10, total_tokens: 70 } }); + res.write('data: [DONE]\n\n'); + res.end(); +} + const server = http.createServer((req, res) => { if (!req.url.includes('/chat/completions')) { res.writeHead(404).end('not found'); @@ -60,6 +72,8 @@ const server = http.createServer((req, res) => { process.stdout = process.stdout; if (hasToolResult) { streamText(res, '文件已创建并验证完毕。'); + } else if (userText.includes('读图')) { + streamReadFileCall(res); } else if (userText.includes('写文件')) { streamToolCall(res); } else if (userText.includes('慢')) { diff --git a/app/test/smoke.js b/app/test/smoke.js index df2556b..ad0f671 100644 --- a/app/test/smoke.js +++ b/app/test/smoke.js @@ -118,6 +118,26 @@ async function main() { ok(!!taskMd, '开始执行自动追加模板消息'); ok(d.session.phase === 'execution', '阶段切换为 execution'); + // ---- 2.5 read_file 读图回传(vision)---- + console.log('\n[2.5] 读图回传'); + const sharp = require('sharp'); + const pngBuf = await sharp({ create: { width: 8, height: 8, channels: 3, background: { r: 200, g: 30, b: 30 } } }).png().toBuffer(); + const si = await api('POST', '/api/sessions', { title: '冒烟-读图' }); + await waitSession(si.id, x => x.session.status === 'active', 5000, '绑定'); + const fdImg = new FormData(); + fdImg.append('files', new Blob([pngBuf], { type: 'image/png' }), 'pic.png'); + const upImg = await fetch(BASE + '/api/sessions/' + si.id + '/files', { method: 'POST', body: fdImg }); + ok(upImg.ok, '上传图片到任务目录'); + await api('POST', '/api/sessions/' + si.id + '/messages', { content: '读图 pic.png' }); + d = await waitSession(si.id, x => x.session.status === 'held' && x.messages.some(m => m.role === 'tool'), 15000, '读图回合完成'); + const imgTool = d.messages.filter(m => m.role === 'tool') + .map(m => { try { return JSON.parse(m.content); } catch (_) { return null; } }) + .find(j => j && Array.isArray(j.__blocks__)); + ok(!!imgTool && imgTool.__blocks__.some(b => + b.type === 'image_url' && String(b.image_url && b.image_url.url).startsWith('data:image/jpeg;base64,')), + 'tool 消息含压缩后的 image_url 内容块', + imgTool ? JSON.stringify(imgTool.__blocks__.map(b => b.type)) : '无多模态块'); + // ---- 3. 路径穿越防护 ---- console.log('\n[3] 路径防护'); let blocked = false; @@ -197,6 +217,15 @@ async function main() { ok(fpath.includes('g.txt'), 'find_path 命中', fpath); const esc = await executeTool(ctx, 'read_file', { path: '../outside.txt' }).catch(e => 'ERR:' + e.message); ok(String(esc).includes('越界'), 'read_file 拒绝越界', String(esc)); + // read_file 读图:非 vision → 文本提示;vision → {text, image_data_url}(依赖 sharp) + const unitPng = await require('sharp')({ create: { width: 4, height: 4, channels: 3, background: { r: 10, g: 120, b: 200 } } }).png().toBuffer(); + fs.writeFileSync(path.join(ctx.taskDir, 'pic.png'), unitPng); + const imgNoVision = await executeTool(ctx, 'read_file', { path: 'pic.png' }); + ok(typeof imgNoVision === 'string' && imgNoVision.includes('不支持视觉识别'), 'read_file 非 vision 图片返回文本提示', String(imgNoVision).slice(0, 80)); + const ctxVision = Object.assign({}, ctx, { vision: true }); + const imgVision = await executeTool(ctxVision, 'read_file', { path: 'pic.png' }); + ok(imgVision && typeof imgVision === 'object' && String(imgVision.image_data_url).startsWith('data:image/jpeg;base64,'), + 'read_file vision 图片返回压缩 data URL', imgVision && imgVision.text ? String(imgVision.text).slice(0, 90) : String(imgVision).slice(0, 90)); // ---- 8. 配置读写 API(阶段 2)---- console.log('\n[8] 配置读写');