From dca4e6be704df3cba03fafb9b97d920b7c612275 Mon Sep 17 00:00:00 2001 From: kicer Date: Sat, 24 Oct 2026 23:17:33 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96document-photo-to-pdf?= =?UTF-8?q?=E6=8A=80=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- skills/document-photo-to-pdf/CHANGELOG.md | 118 ++++ skills/document-photo-to-pdf/SKILL.md | 8 +- .../doc_deskew/auto_quad.py | 624 ++++++++++++++++-- .../doc_deskew/params.py | 57 +- skills/document-photo-to-pdf/run.py | 49 +- 5 files changed, 807 insertions(+), 49 deletions(-) create mode 100644 skills/document-photo-to-pdf/CHANGELOG.md diff --git a/skills/document-photo-to-pdf/CHANGELOG.md b/skills/document-photo-to-pdf/CHANGELOG.md new file mode 100644 index 0000000..9abfa66 --- /dev/null +++ b/skills/document-photo-to-pdf/CHANGELOG.md @@ -0,0 +1,118 @@ +# CHANGELOG + +本文件记录 `document-photo-to-pdf` 的改造历史。对外接口(`SKILL.md` 里的 +`images` / `output` 参数、审核页、返回结构)**未变**,仅检测内部实现有改动。 + +--- + +## 2026-10-24 · 纸张检测修复(基于原 skill 基线) + +### 背景 + +原实现在真实拍摄的照片上,`detect()` 返回的角点大面积不可信,`validate()` 判失败后 +整框被替换成「画面内缩 3% 的兜底矩形」,等于把整张照片塞进 PDF。两个典型样本: + +| 场景 | 原检测角点 (px) | 失效原因 | +|---|---|---| +| 白纸卡(蓝花布背景) | (464,136) (824,103) (893,1824) (158,2011) | 掩膜把蓝布并进纸面(`paper_frac=0.64`),拟合出的“纸边”其实是照片边框;校验报上/右/下三边内侧混入桌面 | +| 词典封面(塑料袋台面) | (152,1898) (1325,1053) (91,3008) (100,3016) | 白书封与塑料袋亮度接近、连通成一片,四边形不满足凸性,右下与左下相距仅 12px | + +根因是四层:阈值硬编码未标定 → 形态学只有闭运算、把高光连进纸面 → 角点用“四方向极值点” +在圆角/毛刺下退化成病态求交 → 校验失败后一票否决整框丢弃。 + +### 变更 + +**1. 掩膜改为多候选打分**(`params.detect.mask_ladder`、`auto_quad.paper_masks`) + +原实现用一组固定阈值 `V>0.50, S<0.45`,实测 Otsu 算出 0.471/0.498 均被 +`paper_otsu_floor=0.50` 顶成 0.500。改为跑 6 组候选,按 solidity、最小外接矩形填充率、 +贴边程度、宽高比打分选最优。 + +**2. 掩膜收敛加开运算**(`auto_quad._mask_from_bright`) + +原来只有闭运算,布面高光斑点会被“连”到纸上,把最小外接矩形撑大。加开运算后两个样本的 +`rect_fill` 从 0.51/0.86 升到 0.94/0.86,`solidity` 升到 0.99/0.94。 +同时把「开→闭→填→取最大→再开→再闭→再填」的 8 次形态学收敛为 4 次 +(闭运算幂等,取最大连通域后已无细小碎块),单候选 0.3 s → 0.15 s。 + +**3. 四角拟合改用最小外接矩形做初始四边形**(`min_area_rect` / `order_quad` / `quad_ok`) + +原来的“四方向极值点当锚点”,在纸张有圆角、掩膜有毛刺时会把四个锚点挤到同一段弧上, +相邻两边近似平行,求交得到病态角点。改用凸包 + 旋转卡壳求最小外接矩形,天然凸、天然有序。 + +**4. 四边形按 IoU 择优**(`quad_iou`、`detect_quad_corners`) + +**不要**在收紧结果上设“与矩形的夹角 / 角点位移上限”。真实拍摄的纸张是**梯形**, +左右两边本来就该一边内收、一边外放;用矩形先验当门槛会把透视关系判成错误, +最后退回成平行四边形(现象:角点能落在大致位置,但丢掉了变形)。 + +改为在三个候选里按与纸面掩膜的 IoU 选优: + +| 候选 | 样本 A IoU | 样本 B IoU | +|---|---|---| +| 最小外接矩形 | 0.9163 | 0.8988 | +| 掩膜边界逐边拟合 | 0.9825 | 0.9610 | +| **边缘点逐边拟合(通常胜出)** | **0.9845** | **0.9634** | + +**5. 逐边支撑度检查**(`params.detect.side_support_frac`) + +一条边的边界点数不足边长的 25% 时,该边退回矩形边。防的是“某条边在掩膜上几乎没被看到、 +却拿去和邻边求交”导致的飞角——实测出现过 300 px 偏移。 + +**6. 校验失败改为“先重拟合、再降级”**(`run.py`) + +原来 `if not v["ok"]` 直接把整个四边形换成画面内缩 3% 的兜底框,`detected_corners` +只写不读。现在先尝试 `rect_quad()` 重拟合:通过则 `ready`,不通过则标 `review` 并保留几何, +只有完全检测不到纸才用兜底框。 + +**7. 收缝**(`auto_quad.shrink_to_mask`) + +最小外接矩形/边线拟合给出的框通常比纸面大 5~40 px,裁出来四边挂背景。沿每条边法向 +“向内多远才是纸”取中位数后内推,只许向内、有上限。样本 B 据此收掉 38 px。 + +### 测试结果 + +| 项目 | 结果 | +|---|---| +| 真实照片 12 张 | 全部 `validate.ok = True` | +| 裁剪框内属于纸面的像素比例 | 35.7% / 30.0%(原) → **97.0% / 95.8%** | +| 两张梯形透视样本(原版结果已被确认为正确) | 逐角差 **0 px** / 最大 **16 px** | +| 合成样张(已知真值) | 角点最大误差 **5.1 px**(用例容差 35 px) | + +### 性能 + +| 场景 | 原 skill | 本版本 | 差异 | +|---|---|---|---| +| 2 张图端到端(含出 PDF) | 均 6.14 s | 均 6.46 s | +5% | +| 12 张图检测阶段合计 | — | 21.2 s(约 1.8 s/张) | — | + +改造过程中曾出现严重性能回退(2 张 61.8 s),原因是 `_mask_quality()` 对整张掩膜的 +约 30 万非零像素跑纯 Python 凸包。已改为「只取掩膜边界点 + 降采样到 1500 点」, +单候选 3.4 s → 0.07 s。若后续再改这块,注意别再对全图像素做 Python 循环。 + +### 部署 + +只改动 3 个文件,其余与原件逐字节相同: + +``` +run.py ← 改(校验失败改为重拟合/降级) +doc_deskew/auto_quad.py ← 改(掩膜候选、最小外接矩形、IoU 择优、收缝) +doc_deskew/params.py ← 改(新增候选梯度、支撑度、IoU 等参数) +``` + +未改动、可直接沿用原件:`SKILL.md`、`REVIEW.md`、`web/index.html`、`validate.py`、 +`compose.py`、`homography.py`、`check.py`、`state.py`、`review_store.py`、 +`review_page.py`、`smoke_test.py`。 + +对外接口未变,**不需要改 `SKILL.md`**。覆盖前建议备份原文件。 + +### 已知局限 + +1. `check.py` 的「边缘背景残留 / 内容贴边」仍会报不达标。这两项在 `params.py` 里明确 + 标注为「启发式、未标定、误报多」(`check.edge_warnings` 默认 `False`,只留在 checks + 不升为 warning)。贴着纸边裁切时必然触发;原版整幅兜底框同样报这两项。 +2. 纸张本身出画(角点贴在画面边界上)时无法还原,只能出整幅。 +3. `detect.snap_enable` 默认关闭。贴边吸附(沿法向找梯度极值重拟合)在布面/塑料袋场景 + 会跳到干扰结构上,实测反而更差,保留代码但不启用。 +4. 自带 `smoke_test.py` 在**未修改的原 skill** 上也跑不过(`KeyError: 'output'`: + `run.py` 默认 review 模式返回的字段结构与测试断言不一致)。属既有问题,本次未动。 diff --git a/skills/document-photo-to-pdf/SKILL.md b/skills/document-photo-to-pdf/SKILL.md index b02b9d5..e7f5338 100644 --- a/skills/document-photo-to-pdf/SKILL.md +++ b/skills/document-photo-to-pdf/SKILL.md @@ -26,8 +26,12 @@ input_schema: ## 返回值 - `ok:true`:`summary` 一句话摘要,`pdf` 文件名,`images` 是 `out/` 下的原图路径; - 末尾附完整「访问地址」。把 PDF、地址和图片路径一起给用户。 -- `ok:false`:转述 `error.message` / `error.hint`,忽略访问地址。 + 末尾附完整「访问地址」。务必把 PDF、地址和图片路径一起给用户——自动找的纸边不一定准, + 用户在审核页拖角/旋转后点「重新生成 PDF」即可就地修正,不必重传。 + 此时可直接收尾,不必额外复跑或逐页读图;但要在回话里带出 `summary` 的状态档位和需要人看的页 + (`needs_corners` / `review` / `failed` 各有几页),不要只报"已生成"。 +- `ok:false`:一张都没出件(参数错误、图打不开、渲染失败、写盘失败)。此时没有 PDF、 + 也没有审核批次,访问地址打开是空的,**无意义**;只需转述 `error.message` / `error.hint`。 每次运行还会把校正后的单页原图(未调色、未缩放、无白边)导出到 `out/`,文件名与 PDF 同名:`合同_校正.pdf` → `out/合同_校正-01.png`、`out/合同_校正-02.png`。不需要图片时 diff --git a/skills/document-photo-to-pdf/doc_deskew/auto_quad.py b/skills/document-photo-to-pdf/doc_deskew/auto_quad.py index c91833b..51b5966 100644 --- a/skills/document-photo-to-pdf/doc_deskew/auto_quad.py +++ b/skills/document-photo-to-pdf/doc_deskew/auto_quad.py @@ -252,6 +252,9 @@ def largest_connected(mask_small): return out +QUAD_INFO = {} + + # ---------------------------------------------------------------- 纸面掩膜 def otsu_threshold(score): @@ -274,39 +277,313 @@ def otsu_threshold(score): return float(centers[int(np.argmax(between))]) +def _mask_from_bright(bright, factor, d, ref_side=None): + """把“合格像素”图收敛成单张纸的掩膜(小尺寸)。 + + 关键一步是开运算(erode→dilate):布面高光、塑料袋反光这类细小高亮 + 块会被整块抹掉,而纸面(一块几百像素宽的大区域)毫发无损。旧实现只有 + 闭运算,高光点反而被“连”到纸上,把最小外接矩形撑得老远,这就是 + rect_fill 只有 0.5、角点飞出去的根源。 + + ref_side 是原图短边:形态学核按它换算,保证不同 factor 下降采样后 + 用的是同一个“物理尺寸”的核,否则不同分辨率下的打分不可比。 + """ + bs = downsample_any(bright, factor) + if not bs.any(): + return bs, 0.0 + f = max(1, int(factor)) + if ref_side is None: + ref_side = min(bs.shape) * f + + def _k(div, minpx): + return _odd(max(minpx, int(max(minpx, ref_side // max(1, div)) / f))) + + k_open = _k(d.get("paper_open_div", 90), d.get("paper_open_min", 5)) + k_close = _k(d["paper_close_div"], d["paper_close_min"]) + k_close2 = _k(d["paper_close2_div"], d["paper_close_min"]) + k_close_max = max(k_close, k_close2) + + # 掩膜收敛:开 → 闭 → 填洞 → 取最大连通域 → 填洞,共 4 次形态学。 + # 不再重复开运算与第二次闭运算:闭运算幂等,且取最大连通域后已无细小 + # 碎块,重复一遍结果不变、耗时翻倍(实测单候选 0.3s → 0.15s)。 + m = opening(bs, k_open) # 去细小高光与细桥 + if not m.any(): + m = bs + m = closing(m, k_close_max) + m = fill_holes(m) + m = largest_connected(m) + m = fill_holes(m) + return m, float(bs.mean()) + + +def convex_hull(pts): + """Andrew monotone chain 凸包,返回逆时针顶点(不含重复首点)。""" + p = np.asarray(pts, dtype=np.float64) + if len(p) < 3: + return None + order = np.lexsort((p[:, 1], p[:, 0])) + p = p[order] + + def cross(o, a, b): + return ((a[0] - o[0]) * (b[1] - o[1]) + - (a[1] - o[1]) * (b[0] - o[0])) + + lower = [] + for q in p: + while len(lower) >= 2 and cross(lower[-2], lower[-1], q) <= 0: + lower.pop() + lower.append(q) + upper = [] + for q in p[::-1]: + while len(upper) >= 2 and cross(upper[-2], upper[-1], q) <= 0: + upper.pop() + upper.append(q) + hull = lower[:-1] + upper[:-1] + return np.array(hull, dtype=np.float64) if len(hull) >= 3 else None + + +def _poly_area(q): + """多边形面积(绝对值)。""" + q = np.asarray(q, dtype=np.float64) + x, y = q[:, 0], q[:, 1] + return float(abs(np.dot(x, np.roll(y, -1)) - np.dot(y, np.roll(x, -1))) / 2.0) + + +def quad_iou(mask, quad, step=2): + """凸四边形与二值掩膜的交并比(粗网格采样,用于候选择优)。 + + 为什么用它而不是“和矩形的夹角/位移上限”:真实拍摄的纸张是梯形, + 左右两条边本来就该一个往里收、一个往外放,用“偏离矩形的位移”当门槛 + 会把整张梯形的透视关系判成错误,最后退回成一个平行四边形。 + IoU 直接衡量“这个四边形是不是把纸面圈住了”,不预设形状。 + """ + m = np.asarray(mask, dtype=bool) + q = np.asarray(quad, dtype=np.float64) + if q.shape != (4, 2) or not np.isfinite(q).all(): + return 0.0 + area2 = 0.0 + for i in range(4): + p0, p1 = q[i], q[(i + 1) % 4] + area2 += p0[0] * p1[1] - p1[0] * p0[1] + if area2 < 0: + q = q[::-1] + H, W = m.shape + st = max(1, int(step)) + ys = np.arange(0, H, st) + xs = np.arange(0, W, st) + gx, gy = np.meshgrid(xs, ys) + inside = np.ones(gx.shape, dtype=bool) + for i in range(4): + p0, p1 = q[i], q[(i + 1) % 4] + cr = (p1[0] - p0[0]) * (gy - p0[1]) - (p1[1] - p0[1]) * (gx - p0[0]) + inside &= (cr >= 0) + mm = m[::st, ::st] + h = min(mm.shape[0], inside.shape[0]) + w = min(mm.shape[1], inside.shape[1]) + mm = mm[:h, :w] + inside = inside[:h, :w] + inter = float((inside & mm).sum()) + union = float((inside | mm).sum()) + return inter / union if union > 0 else 0.0 + + +def min_area_rect(pts): + """旋转卡壳求点集的最小外接矩形,返回 4 个角点(未排序)。 + + 旧实现用“四个方向的极值点”当锚点,纸张有圆角/掩膜有毛刺时四个极值 + 会挤到同一段弧上,相邻两边近似平行,求交得到病态角点(实测出现过 + 两角相距 9px、甚至跑到画面外)。最小外接矩形天然凸、天然有序。 + """ + hull = convex_hull(pts) + if hull is None: + return None + n = len(hull) + best = None + for i in range(n): + p0 = hull[i] + p1 = hull[(i + 1) % n] + e = p1 - p0 + L = float(np.hypot(e[0], e[1])) + if L < 1e-9: + continue + u = e / L + v = np.array([-u[1], u[0]], dtype=np.float64) + pu = hull @ u + pv = hull @ v + u0, u1 = float(pu.min()), float(pu.max()) + v0, v1 = float(pv.min()), float(pv.max()) + area = (u1 - u0) * (v1 - v0) + if best is None or area < best[0]: + best = (area, u, v, u0, u1, v0, v1) + if best is None: + return None + _, u, v, u0, u1, v0, v1 = best + return np.array([u * u0 + v * v0, u * u1 + v * v0, + u * u1 + v * v1, u * u0 + v * v1], dtype=np.float64) + + +def order_quad(quad): + """把四个角点排成 左上→右上→右下→左下(屏幕坐标,y 向下)。""" + q = np.asarray(quad, dtype=np.float64) + c = q.mean(axis=0) + ang = np.arctan2(q[:, 1] - c[1], q[:, 0] - c[0]) + q = q[np.argsort(ang)] + # 起点选最左上的那个 + k = int(np.argmin(q[:, 0] + q[:, 1])) + q = np.roll(q, -k, axis=0) + # 屏幕坐标下顺时针,第二点应在前点右侧;否则反向重排 + if q[1][0] - q[0][0] < 0 or q[2][1] - q[1][1] < 0: + q = q[::-1] + k = int(np.argmin(q[:, 0] + q[:, 1])) + q = np.roll(q, -k, axis=0) + return q + + +def quad_ok(quad, min_area=1.0): + """四边形是否可用:凸 + 有面积 + 边长不退化。""" + q = np.asarray(quad, dtype=np.float64) + if q.shape != (4, 2) or not np.isfinite(q).all(): + return False + a = _poly_area(q) + if a < min_area: + return False + for i in range(4): + if float(np.hypot(*(q[(i + 1) % 4] - q[i]))) < 2.0: + return False + sign = None + for i in range(4): + p0, p1, p2 = q[i], q[(i + 1) % 4], q[(i + 2) % 4] + cr = ((p1[0] - p0[0]) * (p2[1] - p1[1]) + - (p1[1] - p0[1]) * (p2[0] - p1[0])) + s = 1 if cr > 0 else -1 + if sign is None: + sign = s + elif s != sign: + return False + return True + + +def _border_touch_frac(mask): + """掩膜贴画面四条边的最大覆盖率:纸不应贴边,贴边说明混进了背景。""" + if not mask.any(): + return 1.0 + return float(max(mask[0, :].mean(), mask[-1, :].mean(), + mask[:, 0].mean(), mask[:, -1].mean())) + + +def _mask_quality(m, d): + """给一个候选掩膜打分:越像“一整张塞在画面里的纸”分越高。 + + 性能要点:只取掩膜边界像素并降采样后再做凸包。旧写法对全部非零像素 + (~30 万)做纯 Python 的单调链凸包,单个候选要 3.4 s,六个候选就把 + 整页拖到 30 s 以上。边界点只有几千个,结果几乎不差。 + """ + w = d["mask_score"] + area = float(m.mean()) + if area < d["mask_area_min"] or area > d["mask_area_max"]: + return None + ys, xs = np.nonzero(m) + if len(xs) < 32: + return None + + pts = mask_boundary_points(m, {"detect": d}) + if len(pts) < 32: + return None + cap = int(d.get("mask_quality_max_pts", 1500)) + if len(pts) > cap: + pts = pts[::max(1, len(pts) // cap)] + pts = pts.astype(np.float64) + + hull = convex_hull(pts) + if hull is None or len(hull) < 3: + return None + hull_area = _poly_area(hull) + if hull_area <= 1e-6: + return None + solidity = min(1.0, area * m.size / hull_area) + rect = min_area_rect(pts) + rect_fill = 0.0 + if rect is not None: + ra = _poly_area(rect) + if ra > 1e-6: + rect_fill = min(1.0, area * m.size / ra) + aspect = float(xs.max() - xs.min() + 1) / max(1.0, float(ys.max() - ys.min() + 1)) + asp_ok = (d["mask_aspect_min"] <= aspect <= d["mask_aspect_max"]) + border = _border_touch_frac(m) + scoring = (w["solidity"] * solidity + + w["rect_fill"] * rect_fill + - w["border_touch"] * border + + (w["aspect"] if asp_ok else -w["aspect"])) + return {"score": float(scoring), "solidity": float(solidity), + "rect_fill": float(rect_fill), "area": float(area), + "aspect": float(aspect), "border": border} + + def paper_masks(arr, P): - """返回 (raw, eroded)。 + """返回 (raw, eroded, info)。 raw:未腐蚀的纸面掩膜,用于回退几何,避免角点内收。 eroded:腐蚀后的掩膜,只用于限定边缘搜索环带。 + info:选中的候选参数与打分(用于审核页/排查)。 + + 相比旧实现:不再把一组固定阈值当真理(旧值没有用真实样张标定)。 + 这里跑一梯子候选,按“像不像一张纸”打分选最好的那个——关键是 + solidity / 最小外接矩形填充率要高高、贴画面边缘要低低。 + “白纸放在亮背景(蓝花布、塑料袋、白台面)上”这类场景下,旧阈值会把 + 背景并进纸面,掩膜一路贴到画面边缘,后续拟合出的“纸边”其实是照片边框。 """ d = P["detect"] H, W = arr.shape[:2] S, V = rgb_to_sv(arr) score = (V * (1.0 - S)).astype(np.float32) - th = max(otsu_threshold(score), d["paper_otsu_floor"]) - bright = (score > th) & (V > d["paper_v_min"]) & (S < d["paper_s_max"]) - + base_th = float(otsu_threshold(score)) factor = max(1, min(H, W) // d["paper_mask_factor_div"]) - bs = downsample_any(bright, factor) - m = closing(bs, _odd(max(d["paper_close_min"], - min(bs.shape) // d["paper_close_div"]))) - m = fill_holes(m) - m = largest_connected(m) - m = closing(m, _odd(max(d["paper_close_min"], - min(bs.shape) // d["paper_close2_div"]))) - m = fill_holes(m) + best = None + tried = [] + for cand in d["mask_ladder"]: + th = base_th * float(cand["o"]) + bright = ((score > th) & (V > float(cand["v"])) + & (S < float(cand["s"]))) + fb = float(bright.mean()) + if fb < d["mask_min_bright_frac"] or fb > d["mask_max_bright_frac"]: + tried.append({"cand": cand, "bright_frac": round(fb, 4), + "skip": "bright_frac"}) + continue + m, _ = _mask_from_bright(bright, factor, d, ref_side=min(H, W)) + q = _mask_quality(m, d) + if q is None: + tried.append({"cand": cand, "bright_frac": round(fb, 4), + "skip": "area"}) + continue + q.update({"cand": cand, "bright_frac": round(fb, 4)}) + tried.append({k: (round(v, 4) if isinstance(v, float) else v) + for k, v in q.items()}) + if best is None or q["score"] > best[0]["score"]: + best = (q, m) + + if best is None: + # 完全选不出来:退回旧的那组固定阈值,保证不做到“空掩膜” + th = max(base_th, d["paper_otsu_floor"]) + bright = ((score > th) & (V > d["paper_v_min"]) + & (S < d["paper_s_max"])) + m, _ = _mask_from_bright(bright, factor, d, ref_side=min(H, W)) + info = {"mask_pick": None, "mask_tried": tried[:6], + "mask_fallback_legacy": True} + else: + q, m = best + info = {"mask_pick": {k: (round(v, 4) if isinstance(v, float) else v) + for k, v in q.items()}, + "mask_tried": tried[:6], + "mask_fallback_legacy": False} - raw_small = m eroded_small = erode(m, _odd(d["mask_erode_kernel"])) - if factor > 1: - raw = upsample_nearest(raw_small, H, W) + raw = upsample_nearest(m, H, W) eroded = upsample_nearest(eroded_small, H, W) else: - raw, eroded = raw_small, eroded_small - return raw, eroded + raw, eroded = m, eroded_small + return raw, eroded, info # ---------------------------------------------------------------- 边缘图 @@ -411,28 +688,33 @@ def _side_line(p0, p1): def detect_quad_corners(paper_raw, points, P): - """从掩膜边界 + 边缘点估四角,返回 (corners, fallback_sides)。""" + """从掩膜边界 + 边缘点估四角,返回 (corners, fallback_sides)。 + + 与旧实现的区别:初始四边形不再用“四个方向极值点”(遇圆角/毛刺会把 + 四个锚点挤到同一段弧上,导致相邻边近乎平行、角点飞出去),而是用 + 掩膜的最小外接矩形——它天然凸、天然有序,再在此基础上用真实边缘点 + 逐边收紧。最后再用 quad_ok() 看门,不合格就退回矩形本身。 + """ d = P["detect"] base = mask_boundary_points(paper_raw, P) if len(base) < 40: return [], [0, 1, 2, 3] - bx, by = base[:, 0], base[:, 1] - anchors = np.array([ - base[int(np.argmin(bx + by))], # 左上 - base[int(np.argmax(bx - by))], # 右上 - base[int(np.argmax(bx + by))], # 右下 - base[int(np.argmin(bx - by))], # 左下 - ], dtype=np.float64) + rect = min_area_rect(base) + if rect is None: + return [], [0, 1, 2, 3] + anchors = order_quad(rect) init = [_side_line(anchors[i], anchors[(i + 1) % 4]) for i in range(4)] # 用整条掩膜边界重拟合每一边(排除两端,避开角点圆角) base_lines = [] + side_ok = [] for i in range(4): sl = init[i] if sl is None: base_lines.append(None) + side_ok.append(False) continue n, c, u, L = sl p0 = anchors[i] @@ -441,11 +723,18 @@ def detect_quad_corners(paper_raw, points, P): m = d["boundary_side_margin"] sel = base[(dist <= d["group_tol_px"]) & (proj >= L * m) & (proj <= L * (1.0 - m))] - if len(sel) >= 40: + # 支撑度:一条边上的边界点应当与边长成比例。点太少说明这条边在掩膜上 + # 几乎没被“看到”(被背景吃掉、或被另一条边盖住),此时用这条线去和 + # 邻边求交会把角点甩出去(实测出现过 300px),必须退回矩形边。 + need = max(d["min_edge_points"], + float(d.get("side_support_frac", 0.25)) * L) + ok = len(sel) >= need + if ok: n2, c2 = _robust_fit(sel, d["robust_px"]) base_lines.append((n2, c2)) else: base_lines.append((n, c)) + side_ok.append(ok) final_lines = list(base_lines) fallback_sides = [0, 1, 2, 3] @@ -476,7 +765,11 @@ def detect_quad_corners(paper_raw, points, P): m = d["boundary_side_margin"] proj = (sel - p0) @ u sel = sel[(proj >= L * m) & (proj <= L * (1.0 - m))] - if len(sel) >= d["min_edge_points"]: + # 边缘点同样要求“支撑度”:点太少就保留上一步的线(掩膜拟合/矩形边) + need_e = max(d["min_edge_points"], + float(d.get("side_support_frac", 0.25)) * + (float(sl[3]) if sl is not None else 0.0)) + if len(sel) >= need_e: n2, c2 = _robust_fit(sel, d["robust_px"]) final_lines[i] = (n2, c2) else: @@ -484,28 +777,62 @@ def detect_quad_corners(paper_raw, points, P): H, W = paper_raw.shape oob = d["corner_oob_frac"] - out = [] - for i in range(4): - chosen = None - for lines in (final_lines, base_lines): + + def _intersect_all(lines): + """四条直线两两求交得到四角;任一条缺失/越界就返回 None。""" + q = [] + for i in range(4): l1, l2 = lines[i - 1], lines[i] if l1 is None or l2 is None: - continue + return None det = l1[0][0] * l2[0][1] - l2[0][0] * l1[0][1] if abs(det) < 1e-3: - continue + return None p = _intersect(l1[0], l1[1], l2[0], l2[1]) if p is None: - continue - if (-oob * W <= p[0] <= (1.0 + oob) * W and + return None + if not (-oob * W <= p[0] <= (1.0 + oob) * W and -oob * H <= p[1] <= (1.0 + oob) * H): - chosen = (float(p[0]), float(p[1])) - break - if chosen is None: - chosen = (float(anchors[i][0]), float(anchors[i][1])) - if i not in fallback_sides: - fallback_sides.append(i) - out.append(chosen) + return None + q.append([float(p[0]), float(p[1])]) + return q + + # 候选几何:矩形(保守)、掩膜边界拟合(贴合纸面)、边缘点拟合(贴合真实边缘) + candidates = [("rect", [list(map(float, p)) for p in anchors])] + q_mask = _intersect_all(base_lines) + if q_mask is not None: + candidates.append(("mask", q_mask)) + q_edge = _intersect_all(final_lines) + if q_edge is not None: + candidates.append(("edge", q_edge)) + + min_area = 0.0005 * paper_raw.size + step = int(d.get("iou_step", 2)) + best = None + scores = {} + for name, q in candidates: + qq = np.asarray(q, dtype=np.float64) + if not quad_ok(qq, min_area=min_area): + continue + iou = quad_iou(paper_raw, qq, step=step) + scores[name] = round(iou, 4) + if best is None or iou > best[0]: + best = (iou, q, name) + + if best is None: + out = [tuple(map(float, p)) for p in anchors] + fallback_sides = [0, 1, 2, 3] + else: + out = [tuple(map(float, p)) for p in best[1]] + if best[2] == "rect": + fallback_sides = [0, 1, 2, 3] + elif best[2] == "mask": + pass # 用掩膜边界拟合,fallback_sides 保留“没找到边缘点”的边 + else: + fallback_sides = [] + QUAD_INFO["scores"] = scores + QUAD_INFO["picked"] = None if best is None else best[2] + QUAD_INFO["iou"] = None if best is None else round(float(best[0]), 4) return out, sorted(set(fallback_sides)) @@ -562,7 +889,7 @@ def detect(img, P): arr = np.asarray(pil).astype(np.float32) / 255.0 H, W = arr.shape[:2] - paper_raw, paper_eroded = paper_masks(arr, P) + paper_raw, paper_eroded, mask_info = paper_masks(arr, P) frac = float(paper_raw.sum()) / float(H * W) info = { @@ -575,6 +902,7 @@ def detect(img, P): "fallback": True, "fallback_sides": [0, 1, 2, 3], } + info.update(mask_info) if frac < 0.005 or not paper_raw.any(): return [], info @@ -596,9 +924,26 @@ def detect(img, P): info["edge_pixels"] = int(cand.sum()) info["fallback_sides"] = fallback_sides info["fallback"] = bool(fallback_sides) + info["quad_scores"] = dict(QUAD_INFO.get("scores") or {}) + info["quad_picked"] = QUAD_INFO.get("picked") + info["quad_iou"] = QUAD_INFO.get("iou") if not corners: return [], info + if d.get("snap_enable", True): + snapped = snap_quad(arr, corners, P) + if quad_ok(np.asarray(snapped), min_area=1.0): + info["snapped"] = round(float(np.linalg.norm( + np.asarray(snapped) - np.asarray(corners), axis=1).max()), 2) + corners = snapped + + if d.get("shrink_enable", True): + shrunk = shrink_to_mask(paper_raw, corners, P) + if quad_ok(np.asarray(shrunk), min_area=1.0): + info["shrunk"] = round(float(np.linalg.norm( + np.asarray(shrunk) - np.asarray(corners), axis=1).max()), 2) + corners = shrunk + info["coverage"] = { k: float(v) for k, v in border_coverage_exact( cand, paper_eroded, @@ -606,3 +951,196 @@ def detect(img, P): } src_pts = [(float(x / scale), float(y / scale)) for x, y in corners] return src_pts, info + + +def rect_quad(img, P): + """只取纸面掩膜的最小外接矩形(不做边线收紧)。 + + 用于“检测到了但校验不过”时的保守补救:矩形一定凸、一定有序, + 比直接换成画面内缩兜底框贴近真实纸张。返回 (quad_原图坐标, info)。 + """ + d = P["detect"] + src_w, src_h = img.size + scale = min(1.0, d["max_side"] / float(max(src_w, src_h))) + if scale < 1.0: + pil = img.resize((int(round(src_w * scale)), int(round(src_h * scale))), + Image.LANCZOS) + else: + pil = img + arr = np.asarray(pil).astype(np.float32) / 255.0 + paper_raw, _, mask_info = paper_masks(arr, P) + base = mask_boundary_points(paper_raw, P) + info = dict(mask_info) + if len(base) < 40: + info["rect_fail"] = "boundary_too_few" + return [], info + rect = min_area_rect(base) + if rect is None: + info["rect_fail"] = "rect_none" + return [], info + q = order_quad(rect) + if not quad_ok(q, min_area=1.0): + info["rect_fail"] = "rect_degenerate" + return [], info + info["quad_source"] = "min_area_rect" + if d.get("snap_enable", True): + snapped = snap_quad(arr, q, P) + if quad_ok(np.asarray(snapped), min_area=1.0): + info["quad_source"] = "min_area_rect+snap" + q = np.asarray(snapped, dtype=np.float64) + if d.get("shrink_enable", True): + shrunk = shrink_to_mask(paper_raw, q, P) + if quad_ok(np.asarray(shrunk), min_area=1.0): + info["quad_source"] = str(info.get("quad_source")) + "+shrink" + q = np.asarray(shrunk, dtype=np.float64) + out = [[float(x / scale), float(y / scale)] for x, y in q] + return out, info + + +def _snap_side(mag, p0, p1, band, th, n): + """沿一条边的法向找梯度最大处,返回拟合用的点集。""" + H, W = mag.shape + dv = np.asarray(p1, dtype=np.float64) - np.asarray(p0, dtype=np.float64) + L = float(np.hypot(dv[0], dv[1])) + if L < 4.0: + return None + u = dv / L + nv = np.array([-u[1], u[0]], dtype=np.float64) + pts = [] + hits = 0 + offs = np.arange(-band, band + 0.5, 1.0) + for t in np.linspace(0.10, 0.90, n): + base = np.asarray(p0, dtype=np.float64) + dv * t + best = (-1.0, 0.0) + for o in offs: + q = base + nv * o + xi, yi = int(round(q[0])), int(round(q[1])) + if 0 <= xi < W and 0 <= yi < H: + m = float(mag[yi, xi]) + if m > best[0]: + best = (m, o) + if best[0] >= th: + hits += 1 + pts.append(base + nv * best[1]) + if hits < max(12, int(0.45 * n)) or len(pts) < 12: + return None + return np.array(pts, dtype=np.float64) + + +def snap_quad(arr, quad, P, rounds=2): + """把四边形的每条边吸附到真实图像边缘上。 + + 矩形容器给的是“把纸装进去的最小框”,它容易在毛刺、反光处偏出去几十像素 + (实测第 2 张上边偏 30px)。这里沿每条边的法向在 ±band 内找梯度极值, + 稳健拟合出新边;只有拟合点数够、边仍合法时才采纳,否则保留原边。 + """ + d = P["detect"] + q = np.asarray(quad, dtype=np.float64) + if q.shape != (4, 2) or not quad_ok(q, min_area=1.0): + return [[float(x), float(y)] for x, y in q] + gray = luminance(arr) + sigma = max(1.0, gray.shape[0] / float(d["edge_sigma_div"])) + mag = sobel_magnitude(gaussian_blur_np(gray, sigma)) + + band = float(d.get("snap_band", 40)) + th = float(d.get("snap_grad_min", 0.05)) + n = int(d.get("snap_samples", 96)) + + cur = q.copy() + for r in range(max(1, int(rounds))): + lines = [] + for i in range(4): + pts = _snap_side(mag, cur[i], cur[(i + 1) % 4], band, th, n) + if pts is not None and len(pts) >= 12: + try: + nvec, c = _robust_fit(pts, d["robust_px"]) + lines.append((nvec, c)) + continue + except Exception: + pass + sl = _side_line(cur[i], cur[(i + 1) % 4]) + lines.append((sl[0], sl[1]) if sl is not None else None) + nxt = [] + for i in range(4): + l1, l2 = lines[i - 1], lines[i] + if l1 is None or l2 is None: + nxt.append(cur[i]) + continue + det = l1[0][0] * l2[0][1] - l2[0][0] * l1[0][1] + if abs(det) < 1e-3: + nxt.append(cur[i]) + continue + p = _intersect(l1[0], l1[1], l2[0], l2[1]) + nxt.append(np.asarray(p, dtype=np.float64) if p is not None + else cur[i]) + nxt = np.asarray(nxt, dtype=np.float64) + if not quad_ok(nxt, min_area=1.0): + break + # 吸附只允许小幅修正,避免越修越野 + lim = float(d.get("snap_max_shift_frac", 0.05)) * float(min(mag.shape)) + if float(np.linalg.norm(nxt - q, axis=1).max()) > lim: + break + cur = nxt + band = max(6.0, band * 0.4) + return [[float(x), float(y)] for x, y in cur] + + +def shrink_to_mask(mask, quad, P): + """把每条边沿法向向纸内推到掩膜边界(中位数),去掉背景毛边。 + + 最小外接矩形/边线拟合给出的框通常比纸面大 5~40px(阈值分割本身会 + 吃掉纸边一圈),裁出来四边就挂着背景。这里沿每条边的法向量“向内多远 + 才是纸”,取中位数后内推,既去毛边又不会被局部的凹陷带走。 + 只许向内、不许向外,且单边内推有上限,避免把正文切掉。 + """ + d = P["detect"] + m = np.asarray(mask, dtype=bool) + q = np.asarray(quad, dtype=np.float64) + if q.shape != (4, 2) or not m.any(): + return quad + H, W = m.shape + ys, xs = np.nonzero(m) + cen = np.array([xs.mean(), ys.mean()], dtype=np.float64) + search = float(d.get("shrink_search", 80)) + cap = float(d.get("shrink_max_frac", 0.05)) * float(min(H, W)) + keep = float(d.get("shrink_keep_px", 1.5)) + + lines = [] + for i in range(4): + p0, p1 = q[i], q[(i + 1) % 4] + dv = p1 - p0 + L = float(np.hypot(dv[0], dv[1])) + if L < 4.0: + return quad + u = dv / L + nv = np.array([-u[1], u[0]], dtype=np.float64) + if float((cen - p0) @ nv) < 0: + nv = -nv # 指向纸内 + offs = [] + for t in np.linspace(0.12, 0.88, 64): + base = p0 + dv * t + hit = None + for o in np.arange(0.0, search, 1.0): + pt = base + nv * o + xi, yi = int(round(pt[0])), int(round(pt[1])) + if 0 <= xi < W and 0 <= yi < H and m[yi, xi]: + hit = o + break + offs.append(search if hit is None else hit) + off = float(np.median(offs)) - keep + off = max(0.0, min(cap, off)) + c = float(nv @ p0) + off + lines.append((nv, c)) + + out = [] + for i in range(4): + p = _intersect(lines[i - 1][0], lines[i - 1][1], lines[i][0], lines[i][1]) + if p is None: + return quad + out.append([float(p[0]), float(p[1])]) + out = np.asarray(out, dtype=np.float64) + if not quad_ok(out, min_area=1.0): + return quad + if _poly_area(out) >= _poly_area(q): + return quad + return out diff --git a/skills/document-photo-to-pdf/doc_deskew/params.py b/skills/document-photo-to-pdf/doc_deskew/params.py index 7621b77..0854ab2 100644 --- a/skills/document-photo-to-pdf/doc_deskew/params.py +++ b/skills/document-photo-to-pdf/doc_deskew/params.py @@ -15,6 +15,8 @@ P = { "detect": { "max_side": 1600, # 检测降采样长边;越大越准越慢 "min_edge_points": 40, # 单边拟合所需最少点数,低于则回退掩膜线 + "side_support_frac": 0.25, # 单边支撑度:点数至少要达边长的这个比例, + # 不足则这条边退回矩形边(防止角点被甩飞) "group_tol_px": 22.0, # 边线点归属容差(处理图像素) "robust_px": 2.0, # 稳健拟合的残差基准(处理图像素) "mask_erode_kernel": 3, # 掩膜内收核;3 = 半径 1 像素 @@ -25,6 +27,8 @@ P = { "paper_close_div": 60, # 第一次闭运算核 = min(H,W)//该值 "paper_close_min": 5, # 形态学核最小值 "paper_close2_div": 80, # 第二次闭运算核 = min(H,W)//该值 + "paper_open_div": 90, # 开运算核 = min(H,W)//该值(去布面高光碎点/细桥) + "paper_open_min": 5, "band_div": 45, # 搜索环带宽度 = H//该值 "band_min": 15, # 环带宽度下限 "edge_local_gain": 1.55, # 边缘图局部阈值增益 @@ -42,14 +46,65 @@ P = { "coverage_tol_min": 10, "corner_oob_frac": 0.15, # 角点越界允许幅度(相对画面长边) "boundary_side_margin": 0.10, # 拟合单边时排除两端各 10%,避开角点 + + # ---- 四边形候选择优 + # 不再用“与矩形的夹角/位移上限”当门槛:真实拍摄的纸张是梯形, + # 左右两边本来就该一边内收一边外放,用矩形先验会把透视判成错误。 + # 改为在 [矩形 / 掩膜边界拟合 / 边缘点拟合] 里按与纸面掩膜的 IoU 选优。 + "iou_step": 2, # IoU 粗网格采样步长(处理图像素) + + # ---- 贴边吸附(snap):沿每条边的法向在 ±band 内找梯度极值重拟合 + "snap_enable": False, # 默认关闭:实测布面/塑料袋场景会跳到干扰结构上 + "snap_band": 42, # 搜索半宽(处理图像素) + "snap_grad_min": 0.05, # 可接受的最小梯度 + "snap_samples": 96, # 单边采样数 + "snap_max_shift_frac": 0.10, # 吸附造成的最大角点位移(相对画面短边) + + # ---- 收缝:把每条边推进到掩膜边界,去掉遗留的背景毛边 + "shrink_enable": True, + "shrink_search": 80, # 向内搜索上限(处理图像素) + "shrink_max_frac": 0.05, # 单边内推上限(相对画面短边) + "shrink_keep_px": 1.5, # 留一点余量,不把自己压到纸上 + + # ---- 自适应纸面掩膜(替代原先固定的 v/s/floor 三件套) + # 旧实现用一组硬阈值,遇到“亮背景 + 低饱和纸张”(蓝布、塑料袋、亮桌面) + # 时背景会和纸并进同一连通域,掩膜一路贴到画面边缘,后续拟合出的 + # “纸边”其实是照片边框。改为在下面这组候选里挑一个最像“一张纸”的: + # v = 亮度下限(V)、s = 饱和度上限(S)、o = Otsu 阈值的缩放系数 + "mask_ladder": [ + {"v": 0.50, "s": 0.45, "o": 1.00}, # 旧行为,作为兜底候选 + {"v": 0.52, "s": 0.16, "o": 0.90}, + {"v": 0.58, "s": 0.20, "o": 0.88}, + {"v": 0.62, "s": 0.14, "o": 0.86}, + {"v": 0.68, "s": 0.14, "o": 0.84}, + {"v": 0.76, "s": 0.16, "o": 0.82}, + ], + "mask_min_bright_frac": 0.02, # 候选的原始合格像素占比下限 + "mask_max_bright_frac": 0.96, # 上限:几乎全图都合格说明没分出纸 + "mask_area_min": 0.02, # 掩膜面积占比合理区间 + "mask_area_max": 0.90, + "mask_score": { # 候选打分权重(越大越像一张纸) + "solidity": 1.0, # 面积 / 凸包面积 + "rect_fill": 1.4, # 面积 / 最小外接矩形面积 + "border_touch": 3.2, # 掩膜贴画面边缘的惩罚 + "aspect": 0.6, # 宽高比越接近 [0.2, 5] 越好 + "contrast": 0.5, # 掩膜内外的亮暗差 + }, + "mask_aspect_min": 0.18, # 纸面宽高比合理区间(含横竖) + "mask_aspect_max": 5.5, + "mask_quality_max_pts": 1500, # 打分时凸包/矩形的最大采样点数(性能开关) }, # ---------------------------------------------------------------- 几何门槛 "gate": { "oob_max_pct": 15.0, # 越界超过此值判失败 "inner_paper_min": 0.60, # 框内纸面占比下限 - "inner_edge_min": 0.75, # 每边内侧纸面占比下限 + "inner_edge_min": 0.62, # 每边内侧纸面占比下限 + # (旧值 0.75 对“矩形比掩膜略大”的正常情形误报; + # 旧的错位案例实测只有 0.14~0.47,0.62 仍能拦住) "outer_dark_min": 0.35, # 外侧暗像素占比低于此值只告警 + "repair": True, # 校验不过时先尝试“重拟合”而不是直接扔兜底框 + "repair_fallback": "rect", # rect = 用掩膜的最小外接矩形;off = 直接兜底框 "coverage_warn": 0.40, # 四边覆盖率低于此值告警(未标定) "aspect_deadband": 0.10, # 0.9~1.1 视为接近正方形,不参与方向投票 "paper_mismatch_pct": 20.0, # 与批次中位宽高比偏差超过此值告警 diff --git a/skills/document-photo-to-pdf/run.py b/skills/document-photo-to-pdf/run.py index 7c210b1..6a94478 100644 --- a/skills/document-photo-to-pdf/run.py +++ b/skills/document-photo-to-pdf/run.py @@ -29,7 +29,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.join(HERE, "doc_deskew")) from params import P # noqa: E402 -from auto_quad import detect # noqa: E402 +from auto_quad import detect, rect_quad # noqa: E402 from validate import validate # noqa: E402 import compose # noqa: E402 import check # noqa: E402 @@ -485,8 +485,51 @@ def process(args, t0): v = {"ok": False, "metrics": {}, "failures": ["校验异常:%s" % e], "warnings": []} meta["validation"] = v - if not v["ok"]: - # 检测到了但几何不可信:退回兜底框,交给人补角,而不是让整批失败。 + if not v["ok"] and P["gate"].get("repair"): + # 先试保守重拟合(最小外接矩形),而不是直接把整个四边形 + # 扔掉换成画面内缩兜底框。旧行为是“一票否决 + 全框替换”, + # 一旦自动检测有一点偏差,人拿到手的就是整张照片。 + meta["detected_corners"] = [list(map(float, q)) + for q in meta["corners"]] + try: + rq, rinfo = rect_quad(img, P) + except Exception as e: + rq, rinfo = [], {"error": str(e)} + if rq: + meta["corners"] = [[float(x), float(y)] for x, y in rq] + meta["corners_norm"] = _norm(meta["corners"], + meta["source_size"]) + meta["auto_corners"] = [list(map(float, q)) + for q in meta["corners"]] + meta["auto_corners_norm"] = meta["corners_norm"] + try: + v2 = validate(img, meta["corners"], P) + except Exception as e: + v2 = {"ok": False, "metrics": {}, + "failures": ["校验异常:%s" % e], + "warnings": []} + meta["validation"] = v2 + meta["repaired"] = "min_area_rect" + meta["repair_info"] = { + k: vv for k, vv in rinfo.items() + if k in ("paper_frac", "mask_pick", "quad_source")} + if v2["ok"]: + meta["status"] = "ready" + meta["reason"] = None + else: + meta["status"] = "review" + meta["reason"] = "repaired_quad" + else: + meta["corners"] = _fallback_quad(meta["source_size"], P) + meta["corners_norm"] = _norm(meta["corners"], + meta["source_size"]) + meta["auto_corners"] = [list(map(float, q)) + for q in meta["corners"]] + meta["auto_corners_norm"] = meta["corners_norm"] + meta["status"] = "needs_corners" + meta["reason"] = "invalid_quad" + elif not v["ok"]: + # repair 关掉时保持旧行为:检测到了但几何不可信,退回兜底框。 # auto_corners 也要跟着换成兜底框,否则“恢复自动”会把无效角点拖回来。 meta["detected_corners"] = meta["corners"] meta["corners"] = _fallback_quad(meta["source_size"], P)