From 3c9f9d2bdbbef4487c88dee7af82c760fea8527d Mon Sep 17 00:00:00 2001 From: kicer Date: Thu, 19 Nov 2026 19:01:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(device):=20=E6=96=87=E4=BB=B6=E9=80=9A?= =?UTF-8?q?=E9=81=93=20pull/push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit core 与设备复用同一条 WS 搬文件:256KiB 分块、base64、50MiB 上限, 写盘先 .part 再校验 size/sha256 后 rename;device 工具新增 pull/push 两个动作。 --- app/server/devices.js | 128 +++++++++++++++- app/server/tools/definitions.json | 14 +- app/server/tools/executor.js | 34 ++++- device/README.md | 14 ++ device/cmd/agent/file.go | 237 ++++++++++++++++++++++++++++++ device/cmd/agent/main.go | 10 ++ device/pkg/proto/proto.go | 10 ++ 7 files changed, 443 insertions(+), 4 deletions(-) create mode 100644 device/cmd/agent/file.go diff --git a/app/server/devices.js b/app/server/devices.js index f6c3dd1..553aafb 100644 --- a/app/server/devices.js +++ b/app/server/devices.js @@ -7,6 +7,7 @@ // 启用条件:有公网前缀(config.yml public_link.base_url 或 env PUBLIC_BASE_URL) // 且配了 DEVICE_TOKEN。本地部署两者都没有,设备也连不进来,端点干脆不挂。 const crypto = require('crypto'); +const fs = require('fs'); const WebSocket = require('ws'); const log = require('./log'); @@ -106,6 +107,16 @@ function onConnection(ws) { if (p && p.onExit) p.onExit(m); return; } + if (m.t === 'file.chunk') { + const p = state.pending.get(m.id); + if (p && p.onFileChunk) p.onFileChunk(m); + return; + } + if (m.t === 'file.done') { + const p = state.pending.get(m.id); + if (p && p.onFileDone) p.onFileDone(m); + return; + } if (m.t === 'health') { const p = state.pending.get(m.id); if (p && p.onHealth) p.onHealth(m); @@ -273,4 +284,119 @@ function exec({ device, command, cwd, timeout_ms }) { }); } -module.exports = { init, attach, enabled, list, health, exec, PATH }; + +// ---------- 文件通道 ---------- +// core 与设备之间拉/推单个文件,复用同一条 WS:256 KiB 一块,base64 传输。 +// 写盘一律先落 .part,size 与 sha256 都对上才 rename——不留半截文件。 +const FILE_CHUNK = 256 * 1024; +const MAX_FILE = 50 * 1024 * 1024; + +function fileTimeout(bytes) { + return Math.max(61000, Math.ceil(bytes / FILE_CHUNK) * 3000); +} + +// pull:设备 → 会话目录 +function pull({ device, remotePath, localPath }) { + return new Promise((resolve, reject) => { + const dev = pick(device); + if (!dev) return reject(new Error(device ? `设备 ${device} 不在线` : '当前没有设备在线')); + if (!remotePath) return reject(new Error('remote_path 不能为空')); + + const id = newJobId(); + const tmp = localPath + '.part'; + const out = fs.createWriteStream(tmp); + const hash = crypto.createHash('sha256'); + let got = 0; + let settled = false; + + const finish = () => { + if (settled) return false; + settled = true; + clearTimeout(timer); + state.pending.delete(id); + return true; + }; + const fail = msg => { + if (!finish()) return; + try { out.destroy(); } catch (_) {} + try { fs.unlinkSync(tmp); } catch (_) {} + reject(new Error(msg)); + }; + const timer = setTimeout(() => fail('拉取超时(60s 无完成)'), 60000); + + state.pending.set(id, { + onFileChunk(m) { + if (settled) return; + let b; + try { b = Buffer.from(String(m.data || ''), 'base64'); } catch (_) { return fail('数据块解码失败'); } + got += b.length; + if (got > MAX_FILE) return fail('文件超过 50MiB 上限'); + hash.update(b); + out.write(b); + }, + onFileDone(m) { + if (settled) return; + if (m.err) return fail(String(m.err)); + out.end(() => { + if (!finish()) return; + const sum = hash.digest('hex'); + if (typeof m.size === 'number' && m.size !== got) { + try { fs.unlinkSync(tmp); } catch (_) {} + return reject(new Error(`大小不符:收到 ${got},设备报 ${m.size}`)); + } + if (m.sha256 && String(m.sha256).toLowerCase() !== sum) { + try { fs.unlinkSync(tmp); } catch (_) {} + return reject(new Error('sha256 不符')); + } + try { fs.renameSync(tmp, localPath); } catch (e) { return reject(e); } + resolve({ device: dev.name, bytes: got, sha256: sum, path: localPath }); + }); + }, + }); + + dev.ws.send(JSON.stringify({ t: 'file', id, op: 'read', path: remotePath })); + }); +} + +// push:会话目录 → 设备 +async function push({ device, localPath, remotePath }) { + const dev = pick(device); + if (!dev) throw new Error(device ? `设备 ${device} 不在线` : '当前没有设备在线'); + if (!remotePath) throw new Error('remote_path 不能为空'); + + const st = fs.statSync(localPath); + if (!st.isFile()) throw new Error('只能推文件,不能推目录:' + localPath); + if (st.size > MAX_FILE) throw new Error('文件超过 50MiB 上限'); + const buf = fs.readFileSync(localPath); + const sum = crypto.createHash('sha256').update(buf).digest('hex'); + const id = newJobId(); + + const done = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pending.delete(id); + reject(new Error('推送超时')); + }, fileTimeout(buf.length)); + state.pending.set(id, { + onFileDone(m) { + clearTimeout(timer); + state.pending.delete(id); + if (m.err) return reject(new Error(String(m.err))); + resolve({ device: dev.name, bytes: buf.length, sha256: sum, path: remotePath }); + }, + }); + }); + + dev.ws.send(JSON.stringify({ t: 'file', id, op: 'write', path: remotePath, size: buf.length, sha256: sum })); + for (let off = 0; off < buf.length; off += FILE_CHUNK) { + dev.ws.send(JSON.stringify({ + t: 'file.chunk', id, + data: buf.subarray(off, off + FILE_CHUNK).toString('base64'), + })); + // 简单背压:发送缓冲堆到 4MB 就等一等,避免把内存打满 + while (dev.ws.bufferedAmount > 4 * 1024 * 1024) await new Promise(r => setTimeout(r, 10)); + } + dev.ws.send(JSON.stringify({ t: 'file.done', id })); + return done; +} + +module.exports = { init, attach, enabled, list, health, exec, pull, push, PATH }; diff --git a/app/server/tools/definitions.json b/app/server/tools/definitions.json index b3bcc4a..eca129c 100644 --- a/app/server/tools/definitions.json +++ b/app/server/tools/definitions.json @@ -436,7 +436,7 @@ "type": "function", "function": { "name": "device", - "description": "Operate the remote devices (other machines) connected to this server. Devices dial out to this server, so they need no inbound port; they stay connected until closed.\n\n- action=list: no other params. Returns every online device with os/arch/shell/root/health. Always call this first when you don't know device names.\n- action=health: requires device. Returns host uptime, load, memory, disk and temperature (fields the platform can't provide are -1).\n- action=exec: requires device and command. Runs the command through a shell on that device (sh -c on Unix, cmd /c on Windows) and returns stdout, stderr and the exit code. `cwd` must be inside that device's work root; omit it to use the root.\n\nUse it when the work has to happen on the user's own machine (laptop, box, Windows PC). For the machine running this server use `terminal` instead. If a device is offline, say so explicitly — do not silently run the command elsewhere.", + "description": "Operate the remote devices (other machines) connected to this server. Devices dial out to this server, so they need no inbound port; they stay connected until closed.\n\n- action=list: no other params. Returns every online device with os/arch/shell/root/health. Always call this first when you don't know device names.\n- action=health: requires device. Returns host uptime, load, memory, disk and temperature (fields the platform can't provide are -1).\n- action=exec: requires device and command. Runs the command through a shell on that device (sh -c on Unix, cmd /c on Windows) and returns stdout, stderr and the exit code. `cwd` must be inside that device's work root; omit it to use the root.\n\n- action=pull: requires device and remote_path; copies a file from the device into the session directory so read_file / skills can use it.\n- action=push: requires device, local_path and remote_path; sends a session file to the device.\n\nUse it when the work has to happen on the user's own machine (laptop, box, Windows PC). For the machine running this server use `terminal` instead. If a device is offline, say so explicitly — do not silently run the command elsewhere.", "parameters": { "type": "object", "required": [ @@ -448,7 +448,9 @@ "enum": [ "list", "health", - "exec" + "exec", + "pull", + "push" ], "description": "Operation to perform." }, @@ -467,6 +469,14 @@ "timeout_ms": { "type": "integer", "description": "Default 120000, max 600000 (action=exec)." + }, + "remote_path": { + "type": "string", + "description": "设备上的绝对路径,必须在设备工作根内(pull/push)。" + }, + "local_path": { + "type": "string", + "description": "会话目录内的相对路径。pull 省略则用远端文件名;push 必填(源文件)。" } } } diff --git a/app/server/tools/executor.js b/app/server/tools/executor.js index 90b6f18..94604f7 100644 --- a/app/server/tools/executor.js +++ b/app/server/tools/executor.js @@ -834,7 +834,39 @@ async function toolDevice(ctx, args) { } } - return 'Error: action 必须是 list / health / exec'; + if (action === 'pull') { + const name = String(args.device || '').trim(); + const remote = String(args.remote_path || '').trim(); + if (!name) return 'Error: action=pull 需要 device 参数(先调 action=list 看设备名)'; + if (!remote) return 'Error: action=pull 需要 remote_path 参数'; + try { + const rel = String(args.local_path || '').trim() || path.basename(remote); + const abs = resolveInTask(ctx.taskDir, rel, fsRoot(ctx)); + const r = await devices.pull({ device: name, remotePath: remote, localPath: abs }); + recordFile(ctx, abs); + return `已拉到 ${path.relative(ctx.taskDir, abs) || path.basename(abs)}(${r.bytes} 字节,sha256 ${r.sha256.slice(0, 16)}…)`; + } catch (e) { + return 'Error: ' + e.message; + } + } + + if (action === 'push') { + const name = String(args.device || '').trim(); + const local = String(args.local_path || '').trim(); + const remote = String(args.remote_path || '').trim(); + if (!name) return 'Error: action=push 需要 device 参数(先调 action=list 看设备名)'; + if (!local) return 'Error: action=push 需要 local_path 参数(会话目录内的相对路径)'; + if (!remote) return 'Error: action=push 需要 remote_path 参数(设备上的绝对路径,须在工作根内)'; + try { + const abs = resolveInTask(ctx.taskDir, local, fsRoot(ctx)); + const r = await devices.push({ device: name, localPath: abs, remotePath: remote }); + return `已推到 ${name}:${remote}(${r.bytes} 字节,sha256 ${r.sha256.slice(0, 16)}…)`; + } catch (e) { + return 'Error: ' + e.message; + } + } + + return 'Error: action 必须是 list / health / exec / pull / push'; } async function toolTerminal(ctx, args) { diff --git a/device/README.md b/device/README.md index e68e93a..ae76601 100644 --- a/device/README.md +++ b/device/README.md @@ -98,6 +98,7 @@ sh scripts/build.sh all |---|---| | 设备 → core | `{t:"hello", ver, name, os, arch, shell, shell_path, shell_ver, root}` | | core → 设备 | `{t:"exec", id, cmd, cwd, timeout_ms}` · `{t:"kill", id}` · `{t:"ping"}` · `{t:"health", id}` | +| 文件通道 | `{t:"file", id, op:"read"\|"write", path, size?, sha256?}` · `{t:"file.chunk", id, data:}` ×N · `{t:"file.done", id, size?, sha256?, err?}` | | 设备 → core | `{t:"chunk", id, out:"out"\|"err", chunk:}` · `{t:"exit", id, code, killed_by, ms, truncated}` · `{t:"pong"}` · `{t:"health", …}` | - 保活:core 每 20 秒发 `ping`,设备回 `pong`;90 秒无任何消息即断开重连。 @@ -106,6 +107,19 @@ sh scripts/build.sh all - 输出:单流超过上限截断并置 `truncated`。 - `shell` 报的是实际解释器(`/bin/sh` 可能是 dash / bash / busybox),连同版本,供模型决定命令语法。 +## 文件通道 + +core 通过同一条连接搬文件,**设备不需要任何额外端口**: + +| 方向 | 流程 | +|---|---| +| 拉取(设备 → 会话目录) | `file{op:read}` → 设备分块回 `file.chunk` → `file.done{size,sha256}` | +| 推送(会话目录 → 设备) | `file{op:write,size,sha256}` → 分块 `file.chunk` → `file.done` | + +- 每块 256 KiB(base64 后约 341 KiB),单文件上限 50 MiB; +- 写盘先落 `.part`,`size` 与 `sha256` 都对上才 `rename`; +- 设备侧路径必须在工作根内(与 `exec` 的 cwd 校验同一套)。 + ## 安全边界 - 设备主动出站,服务端不需要知道设备 IP; diff --git a/device/cmd/agent/file.go b/device/cmd/agent/file.go new file mode 100644 index 0000000..27a58dd --- /dev/null +++ b/device/cmd/agent/file.go @@ -0,0 +1,237 @@ +package main + +// 文件通道:core 通过同一条 WS 拉取 / 推送文件。 +// +// 协议(与 core 的 devices.js 一一对应): +// +// core → 节点 {t:"file", id, op:"read", path} +// 节点 → core {t:"file.chunk", id, data:} ×N → {t:"file.done", id, size, sha256} +// +// core → 节点 {t:"file", id, op:"write", path, size, sha256} +// core → 节点 {t:"file.chunk", id, data} ×N → {t:"file.done", id} +// 节点 → core {t:"file.done", id, size} 或 {t:"file.done", id, err} +// +// 写盘一律先落 .part,校验通过再 rename——半截文件不能留在原地。 + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + + "agent-device/pkg/wsnode" + "agent-device/pkg/proto" +) + +const ( + fileChunkSize = 256 << 10 // 每块 256 KiB(base64 后约 341 KiB) + maxFileBytes = 50 << 20 // 单文件上限 50 MiB +) + +type fileWriter struct { + path string + tmp *os.File + tmpPath string + hash hash.Hash + got int64 + // 期望值(core 下发时给的,用于对账) + size int64 + sha string +} + +type fileMgr struct { + mu sync.Mutex + writers map[string]*fileWriter +} + +func newFileMgr() *fileMgr { + return &fileMgr{writers: map[string]*fileWriter{}} +} + +// safePath 把设备侧路径限制在 root 之内(与 exec 的 cwd 校验同一套语义)。 +func (r *runner) safePath(raw string) (string, error) { + if raw == "" { + return "", errors.New("path 不能为空") + } + p := raw + if !filepath.IsAbs(p) { + p = filepath.Join(r.root, p) + } + p = filepath.Clean(p) + rel, err := filepath.Rel(r.root, p) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", errors.New("path 超出工作根:" + raw) + } + return p, nil +} + +func (r *runner) fileStart(c *wsnode.Conn, m proto.Msg) { + switch m.Op { + case "read": + go r.fileRead(c, m) + case "write": + r.fileBegin(c, m) + default: + r.fileFail(c, m.ID, "未知的 file.op:"+m.Op) + } +} + +func (r *runner) fileFail(c *wsnode.Conn, id, msg string) { + _ = c.Send(proto.Msg{T: proto.TKindFileDone, ID: id, Err: msg}) +} + +// fileRead 把设备上的文件分块推回 core。 +func (r *runner) fileRead(c *wsnode.Conn, m proto.Msg) { + p, err := r.safePath(m.Path) + if err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + f, err := os.Open(p) + if err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + if st.IsDir() { + r.fileFail(c, m.ID, "是目录,不是文件:"+p) + return + } + if st.Size() > maxFileBytes { + r.fileFail(c, m.ID, "文件超过 50MiB 上限") + return + } + + h := sha256.New() + buf := make([]byte, fileChunkSize) + var sent int64 + for { + n, rerr := f.Read(buf) + if n > 0 { + h.Write(buf[:n]) + sent += int64(n) + if err := c.Send(proto.Msg{ + T: proto.TKindFileChunk, ID: m.ID, + Data: append([]byte(nil), buf[:n]...), + }); err != nil { + return + } + } + if rerr == io.EOF { + break + } + if rerr != nil { + r.fileFail(c, m.ID, rerr.Error()) + return + } + } + _ = c.Send(proto.Msg{ + T: proto.TKindFileDone, ID: m.ID, + Size: sent, SHA: hex.EncodeToString(h.Sum(nil)), + }) +} + +// fileBegin 为一次 write 建立临时文件。 +func (r *runner) fileBegin(c *wsnode.Conn, m proto.Msg) { + p, err := r.safePath(m.Path) + if err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + if m.Size > maxFileBytes { + r.fileFail(c, m.ID, "文件超过 50MiB 上限") + return + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + tmpPath := p + ".part" + f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + r.fileFail(c, m.ID, err.Error()) + return + } + r.files.mu.Lock() + r.files.writers[m.ID] = &fileWriter{ + path: p, tmp: f, tmpPath: tmpPath, hash: sha256.New(), + size: m.Size, sha: m.SHA, + } + r.files.mu.Unlock() +} + +func (r *runner) fileChunk(c *wsnode.Conn, m proto.Msg) { + r.files.mu.Lock() + w := r.files.writers[m.ID] + r.files.mu.Unlock() + if w == nil { + return + } + if w.got+int64(len(m.Data)) > maxFileBytes { + r.abortWriter(m.ID, w, "超过 50MiB 上限") + r.fileFail(c, m.ID, "超过 50MiB 上限") + return + } + if _, err := w.tmp.Write(m.Data); err != nil { + r.abortWriter(m.ID, w, err.Error()) + r.fileFail(c, m.ID, err.Error()) + return + } + w.hash.Write(m.Data) + w.got += int64(len(m.Data)) +} + +// fileEnd 收尾写入:校验大小与 sha256,通过才改名到正式路径。 +func (r *runner) fileEnd(c *wsnode.Conn, m proto.Msg) { + r.files.mu.Lock() + w := r.files.writers[m.ID] + delete(r.files.writers, m.ID) + r.files.mu.Unlock() + if w == nil { + return + } + defer func() { _ = w.tmp.Close() }() + + sum := hex.EncodeToString(w.hash.Sum(nil)) + if w.size > 0 && w.got != w.size { + _ = os.Remove(w.tmpPath) + r.fileFail(c, m.ID, "大小不符:收到 "+strconv.FormatInt(w.got, 10)+",应为 "+strconv.FormatInt(w.size, 10)) + return + } + if w.sha != "" && !strings.EqualFold(w.sha, sum) { + _ = os.Remove(w.tmpPath) + r.fileFail(c, m.ID, "sha256 不符") + return + } + if err := w.tmp.Close(); err != nil { + _ = os.Remove(w.tmpPath) + r.fileFail(c, m.ID, err.Error()) + return + } + if err := os.Rename(w.tmpPath, w.path); err != nil { + _ = os.Remove(w.tmpPath) + r.fileFail(c, m.ID, err.Error()) + return + } + _ = c.Send(proto.Msg{T: proto.TKindFileDone, ID: m.ID, Size: w.got, SHA: sum}) +} + +func (r *runner) abortWriter(id string, w *fileWriter, why string) { + r.files.mu.Lock() + delete(r.files.writers, id) + r.files.mu.Unlock() + _ = w.tmp.Close() + _ = os.Remove(w.tmpPath) +} diff --git a/device/cmd/agent/main.go b/device/cmd/agent/main.go index 870d841..dd74183 100644 --- a/device/cmd/agent/main.go +++ b/device/cmd/agent/main.go @@ -78,6 +78,8 @@ type runner struct { mu sync.Mutex procs map[string]*proc + + files *fileMgr // 文件通道:core 拉取/推送文件时的在途写入 } func main() { @@ -154,6 +156,7 @@ func main() { shell: *shellName, sem: make(chan struct{}, maxParallel), procs: map[string]*proc{}, + files: newFileMgr(), } if r.root == "" { home, _ := os.UserHomeDir() @@ -202,6 +205,13 @@ func main() { } case proto.TKindPing: _ = c.Send(proto.Msg{T: proto.TKindPong}) + case proto.TKindFile: + // write 的建栅必须同步完成:core 紧接着就发 chunk,异步会先到后建、丢块 + r.fileStart(c, m) + case proto.TKindFileChunk: + r.fileChunk(c, m) + case proto.TKindFileDone: + r.fileEnd(c, m) case proto.TKindHealth: h := hostHealth() m := proto.Msg{ diff --git a/device/pkg/proto/proto.go b/device/pkg/proto/proto.go index cd33ce5..9562af8 100644 --- a/device/pkg/proto/proto.go +++ b/device/pkg/proto/proto.go @@ -12,6 +12,9 @@ const ( TKindPing = "ping" // hub → 节点:保活 TKindPong = "pong" // 节点 → hub:保活回应 TKindHealth = "health" // 双向:hub 要指标 / 节点回指标 + TKindFile = "file" // 双向:core 发起一次 read/write + TKindFileChunk = "file.chunk" // 双向:文件数据块(base64) + TKindFileDone = "file.done" // 双向:结束,带 size / sha256 / err ) // Msg 是双向通用信封:不同 t 只用其中一部分字段,其余 omitempty。 @@ -55,4 +58,11 @@ type Msg struct { DiskTotalGB float64 `json:"disk_total_gb,omitempty"` DiskFreeGB float64 `json:"disk_free_gb,omitempty"` TempC float64 `json:"temp_c,omitempty"` + + // file(read/write 共用一个 信封) + Op string `json:"op,omitempty"` // read | write + Path string `json:"path,omitempty"` // 设备侧绝对路径,必须在 root 内 + Size int64 `json:"size,omitempty"` // 文件字节数 + SHA string `json:"sha256,omitempty"` // 文件内容的 sha256(传完对账) + Data []byte `json:"data,omitempty"` // 数据块(JSON 里是 base64) }