houhuan/orc-order-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 百度通用文字识别提取供应商/日期（config api_key 持久化）

Browse Source
This commit is contained in:
houhuan
2026-07-19 19:23:38 +08:00
parent cc66448327
commit 056e7d8e75
14 changed files with 1198 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/batch_service.py
+162
View File
@@ -0,0 +1,162 @@
"""批量处理服务：扫描 data/input/ 下图片，串行跑完整 OCR → 采购单流程。
- 不合并（每张图片单独出一个 result）
- 默认串行（避免百度 OCR 限流）
- 返回 dict 汇总：total / success / failed / results[]
"""
from __future__ import annotations
import logging
import os
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from ..config.settings import ConfigManager
from ..core.utils.log_utils import get_logger
from .ocr_service import OCRService
from .order_service import OrderService
logger = get_logger(__name__)
IMAGE_EXTS = {'.png', '.jpg', '.jpeg', '.bmp', '.webp', '.tif', '.tiff'}
class BatchService:
"""批量处理服务。"""
def __init__(self, config: Optional[ConfigManager] = None):
self.config = config or ConfigManager()
self.ocr_service = OCRService(self.config)
self.order_service = OrderService(self.config)
self._input_folder = self.config.get_path(
'Paths', 'input_folder', fallback='data/input', create=True
)
@property
def input_folder(self) -> str:
return self._input_folder
def list_input_images(self) -> List[str]:
"""列出 data/input/ 下所有待处理图片（按 mtime 升序，先入先出）。"""
if not os.path.isdir(self._input_folder):
return []
files = []
for name in os.listdir(self._input_folder):
p = os.path.join(self._input_folder, name)
if not os.path.isfile(p):
continue
ext = os.path.splitext(name)[1].lower()
if ext in IMAGE_EXTS:
files.append(p)
files.sort(key=lambda p: os.path.getmtime(p))
return files
def process_all_inputs(
self,
progress_cb: Optional[Callable[[int, int, dict], None]] = None,
) -> Dict[str, Any]:
"""批量处理 data/input/ 下全部图片。
Args:
progress_cb: 可选回调 fn(done_count, total_count, latest_result_dict)
Returns:
{
'total': int,
'success': int,
'failed': int,
'results': [
{
'image': str,
'hash': str,
'status': 'success' | 'failed',
'supplier': str,
'bill_date': str,
'total_amount': float,
'result_file': str | None,
'error': str | None,
},
...
]
}
"""
images = self.list_input_images()
total = len(images)
results: List[Dict[str, Any]] = []
if total == 0:
logger.info("data/input/ 下无待处理图片")
return {'total': 0, 'success': 0, 'failed': 0, 'results': []}
logger.info(f"批量处理开始: 共 {total} 张图片")
success = 0
failed = 0
for idx, image_path in enumerate(images, start=1):
entry: Dict[str, Any] = {
'image': image_path,
'hash': '',
'status': 'failed',
'supplier': '',
'bill_date': '',
'total_amount': 0.0,
'result_file': None,
'error': None,
}
try:
file_hash = Path(image_path).stem
entry['hash'] = file_hash
# 1) OCR（写 data/output/{hash}.xlsx + .meta.json）
excel_path = self.ocr_service.process_image(image_path)
if not excel_path:
entry['error'] = 'OCR 失败'
failed += 1
results.append(entry)
if progress_cb:
progress_cb(idx, total, entry)
continue
# 2) process_excel（识别元信息 + 重命名 + 落库）
result_file = self.order_service.process_excel(excel_path)
if not result_file:
entry['error'] = '处理失败'
failed += 1
results.append(entry)
if progress_cb:
progress_cb(idx, total, entry)
continue
# 3) 回查元信息（process_excel 内部已落库）
meta_row = self.order_service.metadata_db.get(file_hash) or {}
entry['result_file'] = result_file
entry['supplier'] = meta_row.get('supplier', '') or ''
entry['bill_date'] = meta_row.get('bill_date', '') or ''
entry['total_amount'] = float(meta_row.get('total_amount') or 0.0)
entry['status'] = 'success'
success += 1
logger.info(
f"[{idx}/{total}] 处理完成: {Path(image_path).name} -> "
f"{Path(result_file).name} | 供应商={entry['supplier']!r} "
f"日期={entry['bill_date']!r} 金额={entry['total_amount']:.2f}"
)
except Exception as e:
logger.error(f"[{idx}/{total}] 处理异常: {image_path}: {e}", exc_info=True)
entry['error'] = str(e)
failed += 1
results.append(entry)
if progress_cb:
progress_cb(idx, total, entry)
summary = {
'total': total,
'success': success,
'failed': failed,
'results': results,
}
logger.info(
f"批量处理完成: 总 {total} 张, 成功 {success}, 失败 {failed}"
)
return summary
app/services/ocr_service.py
+127 -6
View File
@@ -67,6 +67,10 @@ class OCRService:
excel_file = self._get_excel_path(image_path)
if os.path.exists(excel_file):
logger.info(f"文件已处理过，跳过OCR识别: {image_path}")
# 即使 xlsx 已存在，仍补写 meta.json（首次 OCR 后可能未生成）
meta_path = str(excel_file).replace('.xlsx', '.meta.json')
if not os.path.exists(meta_path):
self._write_meta_json(str(excel_file), {}, image_path)
return excel_file
# 执行OCR识别
@@ -74,12 +78,15 @@ class OCRService:
if not result:
logger.error(f"OCR识别失败: {image_path}")
return None
# 生成Excel文件
excel_file = self._generate_excel(result, image_path)
if not excel_file:
logger.error(f"生成Excel文件失败: {image_path}")
return None
# 写 meta.json（从 ocr_client 再做一次通用文字识别获取抬头/日期）
self._write_meta_json(excel_file, {}, image_path)
logger.info(f"处理完成: {image_path} -> {excel_file}")
return excel_file
@@ -171,23 +178,137 @@ class OCRService:
"""
try:
excel_path = self._get_excel_path(image_path)
# 确保输出目录存在
os.makedirs(os.path.dirname(excel_path), exist_ok=True)
# 调用OCR处理器的Excel生成功能
if hasattr(self.ocr_processor, 'generate_excel'):
success = self.ocr_processor.generate_excel(ocr_result, excel_path)
if success:
return excel_path
else:
# 如果OCR处理器没有generate_excel方法，直接返回路径
# 假设OCR处理器已经生成了Excel文件
# OCR 处理器（table_ocr.OCRProcessor）已直接生成 xlsx
if os.path.exists(excel_path):
return excel_path
return None
except Exception as e:
logger.error(f"生成Excel文件时发生错误: {e}", exc_info=True)
return None
def _write_meta_json(self, excel_path: str, ocr_result: dict, image_path: str) -> None:
"""从 OCR 结果中抽取原始文本，写入与 Excel 同名的 .meta.json。
用途：供元信息提取器（OrderMetadataExtractor）使用，无需重新 OCR。
失败不影响主流程。
"""
try:
import json
import base64
from pathlib import Path
base = Path(excel_path)
meta_path = base.with_suffix('.meta.json')
# 0) 优先：调百度通用文字识别（/accurate），覆盖全图文字（含手写抬头/日期）
general_text = ''
general_lines = []
try:
# OCRService.ocr_processor = core.ocr.table_ocr.OCRProcessor
# 其 .ocr_client = BaiduOCRClient (process_image 时初始化)
client = getattr(self.ocr_processor, 'ocr_client', None)
if client is None:
# 显式触发一次 process_image 准备流程 (不会重复 OCR)
try:
self.ocr_processor._ensure_ocr_client()
except Exception:
pass
client = getattr(self.ocr_processor, 'ocr_client', None)
if client and hasattr(client, 'recognize_general') and image_path and os.path.exists(str(image_path)):
words = client.recognize_general(str(image_path))
if words:
# 按 location.top 排序（顶部先），方便后续提取供应商/日期
def _top(w):
loc = w.get('location') or {}
try:
return float(loc.get('top', 0))
except (TypeError, ValueError):
return 0.0
words_sorted = sorted(words, key=_top)
general_lines = [(w.get('words') or '').strip() for w in words_sorted if (w.get('words') or '').strip()]
general_text = '\n'.join(general_lines)
logger.info(f"通用识别获取 {len(general_lines)} 行文字")
except Exception as e:
logger.warning(f"通用识别失败（不影响主流程）: {e}")
# 1) 表格识别 header/body 拼表内文字（已在前面逻辑处理）
ocr_text = ''
ocr_rows = []
for key in ('text', 'raw_text', 'ocr_text', 'content'):
v = ocr_result.get(key)
if isinstance(v, str) and v.strip():
ocr_text = v
break
# 2) 如果没有，尝试从 tables_result 拼
if not ocr_text and isinstance(ocr_result, dict):
tables = ocr_result.get('tables_result') or []
if isinstance(tables, list):
lines = []
for t in tables:
if not isinstance(t, dict):
continue
# 关键：从 header / body / footer 都提取，header 含供应商/日期抬头
for region in ('header', 'body', 'footer'):
region_data = t.get(region) or []
if not isinstance(region_data, list):
continue
for cell in region_data:
if not isinstance(cell, dict):
continue
words = cell.get('words') or cell.get('word') or ''
if not words:
continue
row = cell.get('row') or []
col = cell.get('column') or []
# 按行归一化拼成文本
if isinstance(row, list) and row:
row_key = f"r{row[0]}"
else:
row_key = f"r{len(lines)}"
# 简化：直接每行一个 cell
lines.append(str(words).strip())
ocr_rows.append([str(words)])
if not ocr_text and lines:
ocr_text = '\n'.join(lines)
# 3) 兜底：从 Excel 文件读 cell 拼
if not ocr_text:
try:
import xlrd
rb = xlrd.open_workbook(str(excel_path))
ws = rb.sheet_by_index(0)
lines = []
for r in range(ws.nrows):
row_vals = [str(ws.cell_value(r, c)) for c in range(ws.ncols)]
ocr_rows.append(row_vals)
lines.append(' '.join(row_vals))
ocr_text = '\n'.join(lines)
except Exception as e:
logger.debug(f"从 xlsx 读 OCR 文本失败: {e}")
payload = {
'excel_path': str(excel_path),
'image_path': str(image_path),
'ocr_text': ocr_text,
'ocr_rows': ocr_rows,
'general_text': general_text,
'general_lines': general_lines,
'created_at': __import__('datetime').datetime.now().isoformat(timespec='seconds'),
}
meta_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
logger.debug(f"meta.json 已写入: {meta_path}")
except Exception as e:
logger.warning(f"写 meta.json 失败（不影响主流程）: {e}")
app/services/order_service.py
+163 -2
View File
@@ -5,6 +5,11 @@
"""
import os
import json
import re
import shutil
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
from ..config.settings import ConfigManager
@@ -12,6 +17,8 @@ from ..core.utils.log_utils import get_logger
from ..core.excel.processor import ExcelProcessor
from ..core.excel.merger import PurchaseOrderMerger
from ..core.db.product_db import ProductDatabase
from ..core.db.order_metadata_db import OrderMetadataDB
from ..core.ocr.metadata_extractor import OrderMetadataExtractor, sanitize_for_filename
logger = get_logger(__name__)
@@ -40,7 +47,11 @@ class OrderService:
# 创建Excel处理器和采购单合并器
self.excel_processor = ExcelProcessor(self.config, product_db=self.product_db)
self.order_merger = PurchaseOrderMerger(self.config)
# 元信息识别器 + 单据元信息库
self.extractor = OrderMetadataExtractor()
self.metadata_db = OrderMetadataDB(db_path)
logger.info("OrderService初始化完成")
def get_latest_excel(self) -> Optional[str]:
@@ -89,7 +100,21 @@ class OrderService:
except Exception as e:
logger.error(f"检查特殊预处理时出错: {e}")
return self.excel_processor.process_specific_file(file_path, progress_cb=progress_cb)
result_path = self.excel_processor.process_specific_file(file_path, progress_cb=progress_cb)
if not result_path:
return None
# 应用单据元信息识别 + 重命名 result 与原图
try:
meta = self._extract_and_save_metadata(file_path)
if meta:
result_path = self._apply_metadata_to_filenames(
result_path, file_path, meta
)
except Exception as e:
logger.error(f"应用单据元信息失败（不影响 result 文件）: {e}")
return result_path
def _check_special_preprocess(self, file_path: str) -> Optional[str]:
"""检查并执行特殊的预处理（支持杨碧月、烟草公司、蓉城易购）"""
@@ -243,3 +268,139 @@ class OrderService:
except Exception as e:
logger.error(f"单价校验过程中发生错误: {e}")
return []
# ══════════════════════════════════════════════════════════════
# 单据元信息识别 + 文件重命名
# ══════════════════════════════════════════════════════════════
def _extract_and_save_metadata(self, ocr_excel_path: str) -> Optional[Any]:
"""从 OCR 输出的 xlsx 同目录的 .meta.json 提取元信息，写入 SQLite。
Returns:
OrderMetadata 或 None（失败时）
"""
try:
base = Path(ocr_excel_path)
file_hash = base.stem
meta_path = base.with_suffix('.meta.json')
ocr_text = ''
ocr_rows: List[List[str]] = []
source_image = ''
if meta_path.exists():
try:
payload = json.loads(meta_path.read_text(encoding='utf-8'))
# 优先用通用识别文本（含手写抬头/日期）
ocr_text = payload.get('general_text') or payload.get('ocr_text', '') or ''
ocr_rows = payload.get('ocr_rows', []) or []
source_image = payload.get('image_path', '') or ''
except Exception as e:
logger.warning(f"读 meta.json 失败: {e}")
# 兜底：从 xlsx 拼文本（与 OCRService._write_meta_json 的兜底一致）
if not ocr_text:
try:
import xlrd
rb = xlrd.open_workbook(str(ocr_excel_path))
ws = rb.sheet_by_index(0)
lines = []
for r in range(ws.nrows):
row_vals = [str(ws.cell_value(r, c)) for c in range(ws.ncols)]
ocr_rows.append(row_vals)
lines.append(' '.join(row_vals))
ocr_text = '\n'.join(lines)
except Exception as e:
logger.debug(f"从 xlsx 拼 OCR 文本失败: {e}")
meta = self.extractor.extract(ocr_text, ocr_rows)
self.metadata_db.save(
file_hash=file_hash,
supplier=meta.supplier,
bill_date=meta.bill_date,
total_amount=meta.total_amount,
raw_supplier_text=meta.raw_supplier_text,
source_image=source_image,
)
logger.info(
f"元信息识别: hash={file_hash} supplier={meta.supplier!r} "
f"bill_date={meta.bill_date!r} total_amount={meta.total_amount:.2f}"
)
return meta
except Exception as e:
logger.error(f"_extract_and_save_metadata 失败: {e}")
return None
def _apply_metadata_to_filenames(self, result_path: str,
ocr_excel_path: str,
meta) -> str:
"""应用新文件名规则：
- result: 采购单_{YYYYMMDD}_{供应商}_{hash}.xls
- 原图: {原stem}_{YYYYMMDD}_{供应商}_{hash}.{ext}
任一步骤失败不影响 result 文件本身。
Returns:
新 result 路径（无论重命名是否成功都返回；失败时返回原路径）
"""
try:
file_hash = Path(ocr_excel_path).stem
supplier_clean = sanitize_for_filename(meta.supplier) or '未知供应商'
date_part = meta.bill_date or '未知日期'
# ── 1. result 重命名 ──
new_result_name = f"采购单_{date_part}_{supplier_clean}_{file_hash}.xls"
result_dir = Path(result_path).parent
new_result_path = result_dir / new_result_name
try:
# 冲突时加 _N
if new_result_path.exists() and str(new_result_path) != str(result_path):
new_result_path = self._dedup_path(new_result_path)
if str(new_result_path) != str(result_path):
os.rename(result_path, str(new_result_path))
logger.info(f"result 重命名: {result_path} -> {new_result_path}")
result_path = str(new_result_path)
except Exception as e:
logger.warning(f"result 重命名失败: {e}")
# ── 2. 原图重命名 ──
try:
meta_row = self.metadata_db.get(file_hash)
src_image = (meta_row or {}).get('source_image', '')
if src_image and os.path.exists(src_image):
src_p = Path(src_image)
stem = src_p.stem
ext = src_p.suffix
new_image_name = f"{stem}_{date_part}_{supplier_clean}_{file_hash}{ext}"
new_image_path = src_p.parent / new_image_name
# 不覆盖已重命名的图片
if str(new_image_path) != str(src_p) and not new_image_path.exists():
os.rename(src_p, new_image_path)
logger.info(f"原图重命名: {src_p.name} -> {new_image_path.name}")
# 更新 source_image 路径
self.metadata_db.save(
file_hash=file_hash,
supplier=meta.supplier,
bill_date=meta.bill_date,
total_amount=meta.total_amount,
raw_supplier_text=meta.raw_supplier_text,
source_image=str(new_image_path),
)
except Exception as e:
logger.warning(f"原图重命名失败: {e}")
return result_path
except Exception as e:
logger.error(f"_apply_metadata_to_filenames 失败: {e}")
return result_path
@staticmethod
def _dedup_path(p: Path) -> Path:
"""路径冲突时加 _N 后缀。"""
stem, suffix = p.stem, p.suffix
parent = p.parent
n = 1
while True:
cand = parent / f"{stem}_{n}{suffix}"
if not cand.exists():
return cand
n += 1