feat: 百度通用文字识别提取供应商/日期（config api_key 持久化）
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
"""单据元信息 SQLite 存储。
|
||||
|
||||
独立于 ProductDatabase，与 product_cache.db 共用文件。
|
||||
提供：save / get / list_all / count。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OrderMetadataDB:
|
||||
"""单据元信息库（共用 product_cache.db 文件，独立表 order_metadata）。"""
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS order_metadata (
|
||||
file_hash TEXT PRIMARY KEY,
|
||||
supplier TEXT DEFAULT '',
|
||||
bill_date TEXT DEFAULT '',
|
||||
total_amount REAL DEFAULT 0.0,
|
||||
raw_supplier_text TEXT DEFAULT '',
|
||||
source_image TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT '',
|
||||
updated_at TEXT DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str):
|
||||
self.db_path = db_path
|
||||
self._ensure_table()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
return sqlite3.connect(self.db_path)
|
||||
|
||||
def _ensure_table(self):
|
||||
"""幂等创建表。"""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.db_path) or '.', exist_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute(self.SCHEMA)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save(self, file_hash: str, supplier: str = '', bill_date: str = '',
|
||||
total_amount: float = 0.0, raw_supplier_text: str = '',
|
||||
source_image: str = '') -> bool:
|
||||
"""保存或更新单据元信息（PK = file_hash）。"""
|
||||
if not file_hash:
|
||||
return False
|
||||
now = datetime.now().isoformat(timespec='seconds')
|
||||
conn = self._connect()
|
||||
try:
|
||||
# 使用 UPSERT 语义
|
||||
conn.execute("""
|
||||
INSERT INTO order_metadata
|
||||
(file_hash, supplier, bill_date, total_amount,
|
||||
raw_supplier_text, source_image, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(file_hash) DO UPDATE SET
|
||||
supplier = excluded.supplier,
|
||||
bill_date = excluded.bill_date,
|
||||
total_amount = excluded.total_amount,
|
||||
raw_supplier_text = excluded.raw_supplier_text,
|
||||
source_image = excluded.source_image,
|
||||
updated_at = excluded.updated_at
|
||||
""", (file_hash, supplier, bill_date, total_amount,
|
||||
raw_supplier_text, source_image, now, now))
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"save_order_metadata 失败: {e}")
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get(self, file_hash: str) -> Optional[Dict]:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT * FROM order_metadata WHERE file_hash=?",
|
||||
(file_hash,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def list_all(self, limit: int = 1000) -> List[Dict]:
|
||||
"""列出所有元信息，按 updated_at 倒序。"""
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM order_metadata ORDER BY updated_at DESC LIMIT ?",
|
||||
(limit,)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def count(self) -> int:
|
||||
conn = self._connect()
|
||||
try:
|
||||
return conn.execute("SELECT COUNT(*) FROM order_metadata").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def delete(self, file_hash: str) -> bool:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("DELETE FROM order_metadata WHERE file_hash=?", (file_hash,))
|
||||
conn.commit()
|
||||
return True
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -254,6 +254,58 @@ class BaiduOCRClient:
|
||||
|
||||
logger.error("表格识别失败")
|
||||
return None
|
||||
|
||||
def recognize_general(self, image_data: Union[str, bytes]) -> Optional[List[Dict]]:
|
||||
"""通用文字识别（高精度版）：识别图片整图文字（含表格外的手写抬头/日期等）。
|
||||
|
||||
Returns:
|
||||
[{'words': '...', 'probability': {...}, 'location': {...}}, ...]
|
||||
"""
|
||||
access_token = self.token_manager.get_token()
|
||||
if not access_token:
|
||||
logger.error("无法获取访问令牌，无法进行通用识别")
|
||||
return None
|
||||
|
||||
if isinstance(image_data, str):
|
||||
image_data = self.read_image(image_data)
|
||||
if image_data is None:
|
||||
return None
|
||||
|
||||
url = self.config.get('API', 'general_ocr_url',
|
||||
fallback='https://aip.baidubce.com/rest/2.0/ocr/v1/accurate')
|
||||
url = f"{url}?access_token={access_token}"
|
||||
image_base64 = base64.b64encode(image_data).decode('utf-8')
|
||||
|
||||
payload = {
|
||||
'image': image_base64,
|
||||
'language_type': 'CHN_ENG',
|
||||
'detect_direction': 'true',
|
||||
'probability': 'true',
|
||||
}
|
||||
headers = {'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Accept': 'application/json'}
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
response = requests.post(url, data=payload, headers=headers,
|
||||
timeout=self.timeout)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
if 'error_code' in result:
|
||||
logger.warning(f"通用识别错误: {result.get('error_msg')}")
|
||||
if result.get('error_code') in (110, 111):
|
||||
self.token_manager.refresh_token()
|
||||
return None
|
||||
words_list = result.get('words_result') or []
|
||||
logger.debug(f"通用识别返回 {len(words_list)} 行文字")
|
||||
return words_list
|
||||
logger.warning(f"通用识别请求失败 (尝试 {attempt+1}): {response.text[:200]}")
|
||||
except Exception as e:
|
||||
logger.warning(f"通用识别异常 (尝试 {attempt+1}): {e}")
|
||||
if attempt < self.max_retries - 1:
|
||||
time.sleep(self.retry_delay * (2 ** attempt))
|
||||
logger.error("通用识别失败")
|
||||
return None
|
||||
|
||||
def get_excel_result(self, request_id_or_result: Union[str, Dict]) -> Optional[bytes]:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""单据元信息识别器：从 OCR 文本中提取供应商、单据日期、应付金额。
|
||||
|
||||
识别逻辑：
|
||||
- 供应商：取 OCR 顶部前 20 行，关键词匹配（供货单/供应商/供货方/批发/酒行/商行等）
|
||||
- 日期：全文正则匹配，标准化为 YYYYMMDD
|
||||
- 总金额：关键词优先（应付金额/实付金额/金额合计），兜底取总计行最大金额
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
|
||||
# 供应商关键词（命中后取该行清理结果）
|
||||
SUPPLIER_KEYWORDS = (
|
||||
"供货单", "供应商", "供货方", "批发", "酒行", "商行",
|
||||
"经销", "专卖店", "配送单", "送货单", "采购单", "订单",
|
||||
)
|
||||
|
||||
# 总金额关键词（命中后取该行最近一个金额数字）
|
||||
AMOUNT_KEYWORDS = (
|
||||
"应付金额", "实付金额", "金额合计", "应付合计", "合计金额",
|
||||
"合计", "总计", "总计金额",
|
||||
)
|
||||
|
||||
# 日期正则：4 种格式
|
||||
DATE_PATTERNS = [
|
||||
# 2026年07月17日 / 2026年7月17日
|
||||
re.compile(r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日?'),
|
||||
# 2026-07-17 / 2026/07/17 / 2026.07.17
|
||||
re.compile(r'(\d{4})[-./](\d{1,2})[-./](\d{1,2})'),
|
||||
# 20260717 紧邻 8 位数字
|
||||
re.compile(r'(?<!\d)(\d{4})(\d{2})(\d{2})(?!\d)'),
|
||||
]
|
||||
|
||||
# 金额数字正则（保留 2 位小数或整数）
|
||||
AMOUNT_PATTERN = re.compile(r'(\d{1,7}(?:,\d{3})*(?:\.\d+)?)')
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrderMetadata:
|
||||
supplier: str = ''
|
||||
bill_date: str = '' # YYYYMMDD
|
||||
total_amount: float = 0.0
|
||||
raw_supplier_text: str = ''
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
return bool(self.supplier and self.bill_date)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class OrderMetadataExtractor:
|
||||
"""单据元信息识别器。"""
|
||||
|
||||
# 标题区域行数（采购单抬头 + 日期 + 供应商一般在前 20 行）
|
||||
HEADER_LINE_LIMIT = 20
|
||||
|
||||
# 供应商名最大长度
|
||||
MAX_SUPPLIER_LEN = 50
|
||||
|
||||
def extract(self, ocr_text: str, ocr_rows: Optional[List[List[str]]] = None) -> OrderMetadata:
|
||||
"""从 OCR 原始文本和/或解析后的二维数组提取三字段。
|
||||
|
||||
Args:
|
||||
ocr_text: OCR 原始字符串全文（带换行）
|
||||
ocr_rows: 解析后的二维数组（可选），用于 row-level 精确匹配
|
||||
|
||||
Returns:
|
||||
OrderMetadata
|
||||
"""
|
||||
text = ocr_text or ''
|
||||
supplier, raw_supplier = self._extract_supplier(text, ocr_rows)
|
||||
bill_date = self._extract_bill_date(text)
|
||||
total_amount = self._extract_total_amount(text, ocr_rows)
|
||||
|
||||
return OrderMetadata(
|
||||
supplier=supplier,
|
||||
bill_date=bill_date,
|
||||
total_amount=total_amount,
|
||||
raw_supplier_text=raw_supplier,
|
||||
)
|
||||
|
||||
# ───────────────────────────────────────────────
|
||||
# Supplier
|
||||
# ───────────────────────────────────────────────
|
||||
|
||||
def _extract_supplier(self, text: str, rows: Optional[List[List[str]]]) -> Tuple[str, str]:
|
||||
"""提取供应商名称。返回 (cleaned_name, raw_text)。"""
|
||||
# 1) 关键词匹配：取顶部 N 行
|
||||
header_lines = self._get_header_lines(text, self.HEADER_LINE_LIMIT)
|
||||
for line in header_lines:
|
||||
cleaned = self._clean_supplier_line(line)
|
||||
if not cleaned:
|
||||
continue
|
||||
for kw in SUPPLIER_KEYWORDS:
|
||||
if kw in cleaned:
|
||||
return self._truncate(cleaned), cleaned
|
||||
|
||||
# 2) 兜底：取顶部第一个"含中文且无数字行号"且长度 ≥ 4 的非空行
|
||||
# 但排除"纯日期行"（避免把日期当供应商）
|
||||
for line in header_lines:
|
||||
cleaned = self._clean_supplier_line(line)
|
||||
if not cleaned:
|
||||
continue
|
||||
if not (re.search(r'[\u4e00-\u9fa5]', cleaned) and len(cleaned) >= 4):
|
||||
continue
|
||||
# 排除：纯日期（YYYY-MM-DD / YYYY/MM/DD / YYYY年MM月DD日 / YYYYMMDD）
|
||||
date_only = re.match(r'^\s*[\d年月日/\-\.]+\s*$', cleaned)
|
||||
if date_only:
|
||||
continue
|
||||
# 排除：以"日期"开头的行（属于日期标注）
|
||||
if re.match(r'^\s*日期[:：]?', cleaned):
|
||||
continue
|
||||
# 排除：以"单据"开头的行
|
||||
if re.match(r'^\s*单据[:：]?', cleaned):
|
||||
continue
|
||||
return self._truncate(cleaned), cleaned
|
||||
|
||||
return '', ''
|
||||
|
||||
@staticmethod
|
||||
def _clean_supplier_line(line: str) -> str:
|
||||
"""清理一行文本：去前后空白、去首尾日期/编号/电话/标点。"""
|
||||
s = line.strip()
|
||||
if not s:
|
||||
return ''
|
||||
# 去行首日期/编号前缀
|
||||
s = re.sub(r'^[\s\d\-\.\/年月日:：]+', '', s)
|
||||
# 去行尾标点
|
||||
s = re.sub(r'[\s\.,;:：\-_/\\|]+$', '', s)
|
||||
return s.strip()
|
||||
|
||||
def _truncate(self, s: str) -> str:
|
||||
return s[:self.MAX_SUPPLIER_LEN] if len(s) > self.MAX_SUPPLIER_LEN else s
|
||||
|
||||
# ───────────────────────────────────────────────
|
||||
# BillDate
|
||||
# ───────────────────────────────────────────────
|
||||
|
||||
def _extract_bill_date(self, text: str) -> str:
|
||||
"""提取单据日期，标准化为 YYYYMMDD。"""
|
||||
for pat in DATE_PATTERNS:
|
||||
m = pat.search(text)
|
||||
if m:
|
||||
y, mo, d = m.group(1), m.group(2), m.group(3)
|
||||
if self._is_valid_date(y, mo, d):
|
||||
return f"{y}{int(mo):02d}{int(d):02d}"
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def _is_valid_date(y: str, m: str, d: str) -> bool:
|
||||
try:
|
||||
yi, mi, di = int(y), int(m), int(d)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
if not (1900 <= yi <= 2100):
|
||||
return False
|
||||
if not (1 <= mi <= 12):
|
||||
return False
|
||||
if not (1 <= di <= 31):
|
||||
return False
|
||||
return True
|
||||
|
||||
# ───────────────────────────────────────────────
|
||||
# TotalAmount
|
||||
# ───────────────────────────────────────────────
|
||||
|
||||
def _extract_total_amount(self, text: str, rows: Optional[List[List[str]]]) -> float:
|
||||
"""提取总金额（应付/实付/合计/总计）。"""
|
||||
# 1) 关键词行：取该行最大数字
|
||||
for kw in AMOUNT_KEYWORDS:
|
||||
for line in text.splitlines():
|
||||
if kw in line:
|
||||
nums = AMOUNT_PATTERN.findall(line)
|
||||
if nums:
|
||||
# 取最大的（应付通常在末尾）
|
||||
try:
|
||||
vals = [float(n.replace(',', '')) for n in nums]
|
||||
return max(vals)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# 2) 兜底：取全文最大金额数字（>= 1.0 且含小数点，像金额）
|
||||
all_nums = AMOUNT_PATTERN.findall(text)
|
||||
candidates = []
|
||||
for n in all_nums:
|
||||
try:
|
||||
v = float(n.replace(',', ''))
|
||||
if v >= 1.0 and '.' in n:
|
||||
candidates.append(v)
|
||||
except ValueError:
|
||||
continue
|
||||
return max(candidates) if candidates else 0.0
|
||||
|
||||
# ───────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ───────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _get_header_lines(text: str, n: int) -> List[str]:
|
||||
lines = [l for l in text.splitlines() if l.strip()]
|
||||
return lines[:n]
|
||||
|
||||
|
||||
def sanitize_for_filename(s: str) -> str:
|
||||
"""清理字符串为合法文件名片段：替换 Windows 非法字符为下划线。"""
|
||||
if not s:
|
||||
return ''
|
||||
# Windows 非法字符
|
||||
s = re.sub(r'[\\/:*?"<>|]', '_', s)
|
||||
# 多余空白
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
return s
|
||||
Reference in New Issue
Block a user