perf(device): 大文件支持到 2GiB,超时改空闲判定

- 单文件上限 50MiB → 2GiB(两侧)
- pull 超时改成空闲看门狗:首块 30s,之后每块重置 30s,稳定但慢的链路不再被误杀
- pull 写盘背压:磁盘慢时 ws.pause(),drain 后 resume,结束时兜底恢复(否则连接会永久停住)
- push 改流式读 + 流式算 sha,不再整文件进内存;发完最后一包才等设备确认
This commit is contained in:
2026-11-19 19:22:08 +08:00
parent 3c9f9d2bdb
commit 938a99cabe
3 changed files with 83 additions and 32 deletions
+76 -26
View File
@@ -288,14 +288,28 @@ function exec({ device, command, cwd, timeout_ms }) {
// ---------- 文件通道 ---------- // ---------- 文件通道 ----------
// core 与设备之间拉/推单个文件,复用同一条 WS:256 KiB 一块,base64 传输。 // core 与设备之间拉/推单个文件,复用同一条 WS:256 KiB 一块,base64 传输。
// 写盘一律先落 .part,size 与 sha256 都对上才 rename——不留半截文件。 // 写盘一律先落 .part,size 与 sha256 都对上才 rename——不留半截文件。
const FILE_CHUNK = 256 * 1024; const FILE_CHUNK = 256 * 1024; // 每块 256 KiB
const MAX_FILE = 50 * 1024 * 1024; const MAX_FILE = 2 * 1024 * 1024 * 1024; // 单文件上限 2 GiB
const FIRST_CHUNK_MS = 30000; // 请求发出后等第一块(覆盖设备端 open/stat)
const IDLE_MS = 30000; // 块与块之间的空闲超时:只看卡不卡,不看总共多久
const DONE_WAIT_MS = 60000; // push 发完最后一包后等设备确认
const WS_HIGH = 4 * 1024 * 1024; // 发送缓冲高水位,超了就歇一会
function fileTimeout(bytes) { const sleep = ms => new Promise(r => setTimeout(r, ms));
return Math.max(61000, Math.ceil(bytes / FILE_CHUNK) * 3000);
// 流式算 sha256:2 GiB 不能整块读进内存
function hashFile(p) {
return new Promise((resolve, reject) => {
const h = crypto.createHash('sha256');
fs.createReadStream(p)
.on('data', c => h.update(c))
.on('end', () => resolve(h.digest('hex')))
.on('error', reject);
});
} }
// pull:设备 → 会话目录 // pull:设备 → 会话目录
// 超时按「空闲」算:首块 30s,之后每收到一块重置 30s。稳定但慢的链路不会被误杀。
function pull({ device, remotePath, localPath }) { function pull({ device, remotePath, localPath }) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const dev = pick(device); const dev = pick(device);
@@ -308,12 +322,19 @@ function pull({ device, remotePath, localPath }) {
const hash = crypto.createHash('sha256'); const hash = crypto.createHash('sha256');
let got = 0; let got = 0;
let settled = false; let settled = false;
let paused = false;
let idle = null;
const finish = () => { const finish = () => {
if (settled) return false; if (settled) return false;
settled = true; settled = true;
clearTimeout(timer); clearTimeout(idle);
state.pending.delete(id); state.pending.delete(id);
// 背压可能把连接停在 pause 上:结束时必须恢复,否则后续所有消息都收不到
if (paused) {
paused = false;
try { dev.ws.resume(); } catch (_) {}
}
return true; return true;
}; };
const fail = msg => { const fail = msg => {
@@ -322,20 +343,34 @@ function pull({ device, remotePath, localPath }) {
try { fs.unlinkSync(tmp); } catch (_) {} try { fs.unlinkSync(tmp); } catch (_) {}
reject(new Error(msg)); reject(new Error(msg));
}; };
const timer = setTimeout(() => fail('拉取超时(60s 无完成)'), 60000); const bump = ms => {
clearTimeout(idle);
idle = setTimeout(() => fail(`${ms / 1000}s 没有收到数据,判定卡住`), ms);
};
bump(FIRST_CHUNK_MS);
state.pending.set(id, { state.pending.set(id, {
onFileChunk(m) { onFileChunk(m) {
if (settled) return; if (settled) return;
bump(IDLE_MS);
let b; let b;
try { b = Buffer.from(String(m.data || ''), 'base64'); } catch (_) { return fail('数据块解码失败'); } try { b = Buffer.from(String(m.data || ''), 'base64'); } catch (_) { return fail('数据块解码失败'); }
got += b.length; got += b.length;
if (got > MAX_FILE) return fail('文件超过 50MiB 上限'); if (got > MAX_FILE) return fail('文件超过 2GiB 上限');
hash.update(b); hash.update(b);
out.write(b); // 磁盘比网络慢时暂停从连接读,等 drain 再恢复——否则缓冲会在内存里堆到 OOM
if (!out.write(b) && !paused) {
paused = true;
try { dev.ws.pause(); } catch (_) {}
out.once('drain', () => {
paused = false;
try { dev.ws.resume(); } catch (_) {}
});
}
}, },
onFileDone(m) { onFileDone(m) {
if (settled) return; if (settled) return;
clearTimeout(idle);
if (m.err) return fail(String(m.err)); if (m.err) return fail(String(m.err));
out.end(() => { out.end(() => {
if (!finish()) return; if (!finish()) return;
@@ -359,6 +394,8 @@ function pull({ device, remotePath, localPath }) {
} }
// push:会话目录 → 设备 // push:会话目录 → 设备
// 发送阶段不设总超时(core 自己知道在发,靠 bufferedAmount 控节奏),
// 发完最后一包才开始等设备确认(60s)。
async function push({ device, localPath, remotePath }) { async function push({ device, localPath, remotePath }) {
const dev = pick(device); const dev = pick(device);
if (!dev) throw new Error(device ? `设备 ${device} 不在线` : '当前没有设备在线'); if (!dev) throw new Error(device ? `设备 ${device} 不在线` : '当前没有设备在线');
@@ -366,36 +403,49 @@ async function push({ device, localPath, remotePath }) {
const st = fs.statSync(localPath); const st = fs.statSync(localPath);
if (!st.isFile()) throw new Error('只能推文件,不能推目录:' + localPath); if (!st.isFile()) throw new Error('只能推文件,不能推目录:' + localPath);
if (st.size > MAX_FILE) throw new Error('文件超过 50MiB 上限'); if (st.size > MAX_FILE) throw new Error('文件超过 2GiB 上限');
const buf = fs.readFileSync(localPath); const size = st.size;
const sum = crypto.createHash('sha256').update(buf).digest('hex'); const sum = await hashFile(localPath);
const id = newJobId(); const id = newJobId();
let settled = false;
let doneTimer = null;
let rejectDone = null;
const done = new Promise((resolve, reject) => { const done = new Promise((resolve, reject) => {
const timer = setTimeout(() => { rejectDone = reject;
state.pending.delete(id);
reject(new Error('推送超时'));
}, fileTimeout(buf.length));
state.pending.set(id, { state.pending.set(id, {
onFileDone(m) { onFileDone(m) {
clearTimeout(timer); if (settled) return;
settled = true;
clearTimeout(doneTimer);
state.pending.delete(id); state.pending.delete(id);
if (m.err) return reject(new Error(String(m.err))); if (m.err) return reject(new Error(String(m.err)));
resolve({ device: dev.name, bytes: buf.length, sha256: sum, path: remotePath }); resolve({ device: dev.name, bytes: size, sha256: sum, path: remotePath });
}, },
}); });
}); });
dev.ws.send(JSON.stringify({ t: 'file', id, op: 'write', path: remotePath, size: buf.length, sha256: sum })); try {
for (let off = 0; off < buf.length; off += FILE_CHUNK) { dev.ws.send(JSON.stringify({ t: 'file', id, op: 'write', path: remotePath, size, sha256: sum }));
dev.ws.send(JSON.stringify({ for await (const chunk of fs.createReadStream(localPath, { highWaterMark: FILE_CHUNK })) {
t: 'file.chunk', id, dev.ws.send(JSON.stringify({ t: 'file.chunk', id, data: chunk.toString('base64') }));
data: buf.subarray(off, off + FILE_CHUNK).toString('base64'), while (dev.ws.bufferedAmount > WS_HIGH) await sleep(10);
})); }
// 简单背压:发送缓冲堆到 4MB 就等一等,避免把内存打满 dev.ws.send(JSON.stringify({ t: 'file.done', id }));
while (dev.ws.bufferedAmount > 4 * 1024 * 1024) await new Promise(r => setTimeout(r, 10)); } catch (e) {
settled = true;
state.pending.delete(id);
throw e;
} }
dev.ws.send(JSON.stringify({ t: 'file.done', id }));
// 发完最后一包才开始看门狗:设备还要算 sha、rename,慢盘上会停几秒
doneTimer = setTimeout(() => {
if (settled) return;
settled = true;
state.pending.delete(id);
rejectDone(new Error(`发完最后一包后 ${DONE_WAIT_MS / 1000}s 仍未收到设备确认`));
}, DONE_WAIT_MS);
return done; return done;
} }
+2 -1
View File
@@ -116,7 +116,8 @@ core 通过同一条连接搬文件,**设备不需要任何额外端口**:
| 拉取(设备 → 会话目录) | `file{op:read}` → 设备分块回 `file.chunk` → `file.done{size,sha256}` | | 拉取(设备 → 会话目录) | `file{op:read}` → 设备分块回 `file.chunk` → `file.done{size,sha256}` |
| 推送(会话目录 → 设备) | `file{op:write,size,sha256}` → 分块 `file.chunk` → `file.done` | | 推送(会话目录 → 设备) | `file{op:write,size,sha256}` → 分块 `file.chunk` → `file.done` |
- 每块 256 KiB(base64 后约 341 KiB),单文件上限 50 MiB; - 每块 256 KiB(base64 后约 341 KiB),单文件上限 2 GiB;
- 超时按**空闲**算:首块 30s,之后每收到一块重置 30s——链路慢但不卡就不会被误杀;
- 写盘先落 `<path>.part`,`size` 与 `sha256` 都对上才 `rename`; - 写盘先落 `<path>.part`,`size` 与 `sha256` 都对上才 `rename`;
- 设备侧路径必须在工作根内(与 `exec` 的 cwd 校验同一套)。 - 设备侧路径必须在工作根内(与 `exec` 的 cwd 校验同一套)。
+5 -5
View File
@@ -31,7 +31,7 @@ import (
const ( const (
fileChunkSize = 256 << 10 // 每块 256 KiB(base64 后约 341 KiB) fileChunkSize = 256 << 10 // 每块 256 KiB(base64 后约 341 KiB)
maxFileBytes = 50 << 20 // 单文件上限 50 MiB maxFileBytes = 2 << 30 // 单文件上限 2 GiB
) )
type fileWriter struct { type fileWriter struct {
@@ -110,7 +110,7 @@ func (r *runner) fileRead(c *wsnode.Conn, m proto.Msg) {
return return
} }
if st.Size() > maxFileBytes { if st.Size() > maxFileBytes {
r.fileFail(c, m.ID, "文件超过 50MiB 上限") r.fileFail(c, m.ID, "文件超过 2GiB 上限")
return return
} }
@@ -151,7 +151,7 @@ func (r *runner) fileBegin(c *wsnode.Conn, m proto.Msg) {
return return
} }
if m.Size > maxFileBytes { if m.Size > maxFileBytes {
r.fileFail(c, m.ID, "文件超过 50MiB 上限") r.fileFail(c, m.ID, "文件超过 2GiB 上限")
return return
} }
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
@@ -180,8 +180,8 @@ func (r *runner) fileChunk(c *wsnode.Conn, m proto.Msg) {
return return
} }
if w.got+int64(len(m.Data)) > maxFileBytes { if w.got+int64(len(m.Data)) > maxFileBytes {
r.abortWriter(m.ID, w, "超过 50MiB 上限") r.abortWriter(m.ID, w, "超过 2GiB 上限")
r.fileFail(c, m.ID, "超过 50MiB 上限") r.fileFail(c, m.ID, "超过 2GiB 上限")
return return
} }
if _, err := w.tmp.Write(m.Data); err != nil { if _, err := w.tmp.Write(m.Data); err != nil {