houhuan/orc-order-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update: Refactor to absolute imports, fix QPS limits, and enhance Gitea sync with SQLite support

Browse Source
This commit is contained in:
houhuan
2026-07-21 10:25:29 +08:00
parent ef04fc5627
commit 97d9d98b4c
67 changed files with 1691 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/__init__.py
View File
app/core/db/product_db.py
+62 -3
View File
@@ -16,9 +16,9 @@ from typing import Dict, List, Optional, Tuple, Callable
import pandas as pd
from ..utils.log_utils import get_logger
from ..utils.file_utils import smart_read_excel
from ...core.handlers.column_mapper import ColumnMapper
from app.core.utils.log_utils import get_logger
from app.core.utils.file_utils import smart_read_excel
from app.core.handlers.column_mapper import ColumnMapper
logger = get_logger(__name__)
@@ -43,6 +43,13 @@ class ProductDatabase:
max_price REAL DEFAULT 0.0,
price_count INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS missing_barcodes (
barcode TEXT PRIMARY KEY,
name TEXT DEFAULT '',
last_seen TEXT,
source_file TEXT,
count INTEGER DEFAULT 1
);
"""
_NEW_COLUMNS = {
@@ -90,16 +97,68 @@ class ProductDatabase:
def _migrate_schema(self):
conn = self._connect()
try:
# 迁移 products 表
cursor = conn.execute("PRAGMA table_info(products)")
existing_cols = {row[1] for row in cursor.fetchall()}
for col_name, col_type in self._NEW_COLUMNS.items():
if col_name not in existing_cols:
conn.execute(f"ALTER TABLE products ADD COLUMN {col_name} {col_type}")
logger.info(f"数据库迁移: 添加列 {col_name}")
# 确保 missing_barcodes 表存在
conn.execute("""
CREATE TABLE IF NOT EXISTS missing_barcodes (
barcode TEXT PRIMARY KEY,
name TEXT DEFAULT '',
last_seen TEXT,
source_file TEXT,
count INTEGER DEFAULT 1
)
""")
conn.commit()
finally:
conn.close()
# ══════════════════════════════════════════════════════════════
# 缺失条码记录
# ══════════════════════════════════════════════════════════════
def record_missing_barcode(self, barcode: str, name: str = '', source_file: str = ''):
"""记录缺失条码。"""
barcode = str(barcode).strip()
if not barcode:
return
now = datetime.now().isoformat(timespec='seconds')
conn = self._connect()
try:
conn.execute("""
INSERT INTO missing_barcodes (barcode, name, last_seen, source_file, count)
VALUES (?, ?, ?, ?, 1)
ON CONFLICT(barcode) DO UPDATE SET
name = CASE WHEN excluded.name != '' THEN excluded.name ELSE name END,
last_seen = excluded.last_seen,
source_file = excluded.source_file,
count = count + 1
""", (barcode, name, now, os.path.basename(source_file)))
conn.commit()
logger.info(f"已记录缺失条码: {barcode} ({name})")
except Exception as e:
logger.error(f"记录缺失条码失败: {e}")
finally:
conn.close()
def get_missing_barcodes(self, limit: int = 100) -> List[Dict]:
"""获取缺失条码列表。"""
conn = self._connect()
conn.row_factory = sqlite3.Row
try:
rows = conn.execute(
"SELECT * FROM missing_barcodes ORDER BY last_seen DESC LIMIT ?",
(limit,)).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
# ══════════════════════════════════════════════════════════════
# 导入
# ══════════════════════════════════════════════════════════════
app/core/excel/converter.py
+4 -4
View File
@@ -10,13 +10,13 @@ import os
import json
from typing import Dict, Tuple, Optional, Any, List, Union
from ..utils.log_utils import get_logger
from .handlers.barcode_mapper import BarcodeMapper
from .handlers.unit_converter_handlers import (
from app.core.utils.log_utils import get_logger
from app.core.excel.handlers.barcode_mapper import BarcodeMapper
from app.core.excel.handlers.unit_converter_handlers import (
JianUnitHandler, BoxUnitHandler, TiHeUnitHandler,
GiftUnitHandler, UnitHandler
)
from .validators import ProductValidator
from app.core.excel.validators import ProductValidator
logger = get_logger(__name__)
app/core/excel/handlers/__init__.py
+2 -2
View File
@@ -7,5 +7,5 @@
from typing import Dict, Any
# 导出所有处理程序类
from .barcode_mapper import BarcodeMapper
from .unit_converter_handlers import JianUnitHandler, BoxUnitHandler, TiHeUnitHandler, GiftUnitHandler, UnitHandler
from app.core.excel.handlers.barcode_mapper import BarcodeMapper
from app.core.excel.handlers.unit_converter_handlers import JianUnitHandler, BoxUnitHandler, TiHeUnitHandler, GiftUnitHandler, UnitHandler
app/core/excel/handlers/barcode_mapper.py
+1 -1
View File
@@ -7,7 +7,7 @@
import logging
from typing import Dict, Optional, Any
from ...utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/excel/handlers/unit_converter_handlers.py
+1 -1
View File
@@ -8,7 +8,7 @@ import logging
from typing import Dict, Optional, Any, Tuple, Protocol
from abc import ABC, abstractmethod
from ...utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/excel/merger.py
+5 -5
View File
@@ -14,17 +14,17 @@ from xlutils.copy import copy as xlcopy
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
from datetime import datetime
from ...config.settings import ConfigManager
from ..utils.log_utils import get_logger
from ..handlers.column_mapper import ColumnMapper
from ..utils.file_utils import (
from app.config.settings import ConfigManager
from app.core.utils.log_utils import get_logger
from app.core.handlers.column_mapper import ColumnMapper
from app.core.utils.file_utils import (
ensure_dir,
get_file_extension,
get_files_by_extensions,
load_json,
save_json
)
from ..utils.string_utils import (
from app.core.utils.string_utils import (
clean_string,
clean_barcode,
format_barcode
app/core/excel/processor.py
+39 -9
View File
@@ -14,23 +14,23 @@ from xlutils.copy import copy as xlcopy
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
from datetime import datetime
from ...config.settings import ConfigManager
from ..utils.log_utils import get_logger
from ..utils.file_utils import (
from app.config.settings import ConfigManager
from app.core.utils.log_utils import get_logger
from app.core.utils.file_utils import (
ensure_dir,
get_file_extension,
get_latest_file,
load_json,
save_json
)
from ..utils.string_utils import (
from app.core.utils.string_utils import (
clean_string,
extract_number,
format_barcode,
parse_monetary_string
)
from .converter import UnitConverter
from ..handlers.column_mapper import ColumnMapper
from app.core.excel.converter import UnitConverter
from app.core.handlers.column_mapper import ColumnMapper
logger = get_logger(__name__)
@@ -40,15 +40,19 @@ class ExcelProcessor:
提取条码、单价和数量，并按照采购单模板的格式填充
"""
def __init__(self, config, product_db=None):
def __init__(self, config, product_db=None, missing_barcodes_cb: Optional[Callable[[List[str]], None]] = None):
"""
初始化Excel处理器
Args:
config: 配置信息
product_db: 商品数据库实例（可选，由外部传入以共享）
missing_barcodes_cb: 缺失条码的回调函数，接收条码列表
"""
self.config = config
self.missing_barcodes_cb = missing_barcodes_cb
self.current_missing_barcodes = [] # 记录当前文件处理中缺失的条码
self.current_file_path = "" # 记录当前处理的文件路径
# 修复ConfigParser对象没有get_path方法的问题
try:
@@ -62,7 +66,7 @@ class ExcelProcessor:
logger.warning(f"模板文件不存在: {self.template_path}")
# 设置缓存文件路径
self.cache_file = os.path.join(self.output_dir, "processed_files.json")
self.cache_file = os.path.join(self.output_dir, "excel_process_records.json")
self.processed_files = self._load_processed_files()
# 确保目录存在
@@ -80,7 +84,7 @@ class ExcelProcessor:
if product_db is not None:
self.product_db = product_db
else:
from ..db.product_db import ProductDatabase
from app.core.db.product_db import ProductDatabase
db_path = config.get_path('Paths', 'product_db', fallback='data/product_cache.db') if hasattr(config, 'get_path') else 'data/product_cache.db'
tpl_folder = config.get('Paths', 'template_folder', fallback='templates')
item_data = config.get('Templates', 'item_data', fallback='商品资料.xlsx')
@@ -220,6 +224,21 @@ class ExcelProcessor:
# 跳过空条码行
if not product['barcode']:
continue
# 检查条码是否存在于数据库（商品资料）
bc = product['barcode']
mem = self.product_db.get_memory(bc)
if not mem or mem.get('confidence', 0) < 50:
# 如果不存在，或者置信度低（说明不是来自商品资料模板），记录为缺失
if bc not in self.current_missing_barcodes:
self.current_missing_barcodes.append(bc)
# 记录到数据库
self.product_db.record_missing_barcode(
bc,
product.get('name', ''),
self.current_file_path
)
logger.warning(f"条码缺失: {bc} ({product.get('name', '')})")
# 检查备注列，过滤换货、退货、作废等非采购行
skip_row = False
@@ -606,6 +625,9 @@ class ExcelProcessor:
if not os.path.exists(file_path):
logger.error(f"文件不存在: {file_path}")
return None
self.current_missing_barcodes = [] # 重置缺失列表
self.current_file_path = file_path # 设置当前处理文件路径
try:
# 读取Excel文件时不立即指定表头
@@ -672,6 +694,14 @@ class ExcelProcessor:
# 不再自动打开输出目录
logger.info(f"采购单已保存到: {output_file}")
# 处理完成，如果有缺失条码，触发回调
if self.current_missing_barcodes and self.missing_barcodes_cb:
try:
self.missing_barcodes_cb(self.current_missing_barcodes)
except Exception as e:
logger.error(f"触发缺失条码回调失败: {e}")
if progress_cb:
try:
progress_cb(100)
app/core/excel/validators.py
+2 -2
View File
@@ -8,8 +8,8 @@ import re
import logging
from typing import Dict, Any, Optional, List, Tuple, Union
from ..utils.log_utils import get_logger
from ..utils.string_utils import parse_monetary_string
from app.core.utils.log_utils import get_logger
from app.core.utils.string_utils import parse_monetary_string
logger = get_logger(__name__)
app/core/handlers/__init__.py
+3 -3
View File
@@ -2,8 +2,8 @@
数据处理handlers模块初始化文件
"""
from .data_cleaner import DataCleaner
from .column_mapper import ColumnMapper
from .calculator import DataCalculator
from app.core.handlers.data_cleaner import DataCleaner
from app.core.handlers.column_mapper import ColumnMapper
from app.core.handlers.calculator import DataCalculator
__all__ = ['DataCleaner', 'ColumnMapper', 'DataCalculator']
app/core/handlers/calculator.py
+1 -1
View File
@@ -7,7 +7,7 @@
import pandas as pd
import numpy as np
from typing import Dict, Any, Optional, List, Union
from ...core.utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/handlers/column_mapper.py
+3 -3
View File
@@ -7,7 +7,7 @@
import re
import pandas as pd
from typing import Dict, Any, Optional, List, Union
from ...core.utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
@@ -22,7 +22,7 @@ class ColumnMapper:
STANDARD_COLUMNS = {
'barcode': [
'条码', '条形码', '商品条码', '商品条形码', '产品条码', '商品编码',
'商品编号', '条码（必填）', '电脑条码', '条码ID',
'商品编号', '条码（必填）', '电脑条码', '条码ID', '单品条码',
'barcode', 'Barcode', 'BarCode', 'code', '编码',
],
'name': [
@@ -363,7 +363,7 @@ class ColumnMapper:
"""
header_keywords = [
'条码', '条形码', '商品条码', '商品名称', '名称', '规格',
'单价', '数量', '金额', '单位', '必填', '编码',
'单价', '数量', '金额', '单位', '必填', '编码', '单品条码', '序号',
]
best_row = -1
app/core/handlers/data_cleaner.py
+1 -1
View File
@@ -6,7 +6,7 @@
import pandas as pd
from typing import Dict, Any, Optional, List, Union
from ...core.utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/ocr/baidu_ocr.py
+127 -4
View File
@@ -9,7 +9,7 @@ import base64
import requests
from typing import Dict, Optional, Union, List
from ..utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
@@ -229,10 +229,19 @@ class BaiduOCRClient:
logger.debug(f"百度OCR API返回结果: {result}")
if 'error_code' in result:
error_code = result.get('error_code')
error_msg = result.get('error_msg', '未知错误')
# 如果是 QPS 限制 (18)，增加延迟并重试
if error_code == 18:
wait_time = self.retry_delay * (attempt + 1)
logger.warning(f"触发 QPS 限制，将在 {wait_time} 秒后重试 (尝试 {attempt+1}/{self.max_retries})")
time.sleep(wait_time)
continue
logger.error(f"百度OCR API错误: {error_msg}")
# 如果是授权错误，尝试刷新令牌
if result.get('error_code') in [110, 111]: # 授权相关错误码
if error_code in [110, 111]: # 授权相关错误码
logger.info("尝试刷新访问令牌...")
self.token_manager.refresh_token()
return None
@@ -292,8 +301,18 @@ class BaiduOCRClient:
if response.status_code == 200:
result = response.json()
if 'error_code' in result:
logger.warning(f"通用识别错误: {result.get('error_msg')}")
if result.get('error_code') in (110, 111):
error_code = result.get('error_code')
error_msg = result.get('error_msg', '未知错误')
# 如果是 QPS 限制 (18)，增加延迟并重试
if error_code == 18:
wait_time = self.retry_delay * (attempt + 1)
logger.warning(f"触发 QPS 限制 (General)，将在 {wait_time} 秒后重试 (尝试 {attempt+1}/{self.max_retries})")
time.sleep(wait_time)
continue
logger.warning(f"通用识别错误: {error_msg}")
if error_code in (110, 111):
self.token_manager.refresh_token()
return None
words_list = result.get('words_result') or []
@@ -306,6 +325,110 @@ class BaiduOCRClient:
time.sleep(self.retry_delay * (2 ** attempt))
logger.error("通用识别失败")
return None
def recognize_ticket(self, image_data: Union[str, bytes]) -> Optional[List[Dict]]:
"""通用票务识别：用于精准捕获表头抬头、日期等关键信息。
使用用户指定的 URL: https://aip.baidubce.com/rest/2.0/ocr/v1/general_ocr
Returns:
[{'words': '...'}, ...]
"""
access_token = self.token_manager.get_token()
if not access_token:
logger.error("无法获取访问令牌，无法进行票务识别")
return None
if isinstance(image_data, str):
image_data = self.read_image(image_data)
if image_data is None:
return None
# 使用用户指定的通用票务识别接口
url = self.config.get('API', 'ticket_ocr_url',
fallback='https://aip.baidubce.com/rest/2.0/ocr/v1/general_ocr')
url = f"{url}?access_token={access_token}"
image_base64 = base64.b64encode(image_data).decode('utf-8')
payload = {
'image': image_base64,
'detect_direction': 'true', # 开启方向检测，应对旋转的图片
}
headers = {'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'}
for attempt in range(self.max_retries):
try:
response = requests.post(url, data=payload, headers=headers,
timeout=self.timeout)
if response.status_code == 200:
result = response.json()
if 'error_code' in result:
error_code = result.get('error_code')
error_msg = result.get('error_msg', '未知错误')
# 如果是 QPS 限制 (18)，增加延迟并重试
if error_code == 18:
wait_time = self.retry_delay * (attempt + 1)
logger.warning(f"触发 QPS 限制 (Ticket)，将在 {wait_time} 秒后重试 (尝试 {attempt+1}/{self.max_retries})")
time.sleep(wait_time)
continue
# 如果是权限错误 (6)，记录详细日志告知用户
if error_code == 6:
logger.error("权限错误 (6): 您的百度 API Key 未开启【通用卡证票据识别】(General OCR) 服务。请前往百度云控制台手动开启该服务，否则无法精准识别表头。")
logger.warning(f"票务识别错误: {error_msg}")
if error_code in (110, 111):
self.token_manager.refresh_token()
return None
# 1. 优先处理 results 结构 (通用卡证票据识别的新结构)
if 'results' in result and isinstance(result['results'], dict):
extracted = []
# 通常只有一个结果 "0"
for res_id, res_content in result['results'].items():
if not isinstance(res_content, dict): continue
for k, v_list in res_content.items():
if isinstance(v_list, list):
for item in v_list:
if isinstance(item, dict) and 'words' in item:
words_val = item['words']
if isinstance(words_val, list):
for w in words_val:
extracted.append({'words': f"{k}: {w}"})
else:
extracted.append({'words': f"{k}: {words_val}"})
return extracted
# 2. 通用票据/票务识别通常也返回 words_result
words_list = result.get('words_result')
# 如果返回的是结构化字典，提取其中的文字行
if isinstance(words_list, dict):
extracted = []
for k, v in words_list.items():
if isinstance(v, dict) and 'words' in v:
words_val = v['words']
if isinstance(words_val, list):
for w in words_val:
extracted.append({'words': f"{k}: {w}"})
else:
extracted.append({'words': f"{k}: {words_val}"})
elif isinstance(v, str):
extracted.append({'words': f"{k}: {v}"})
return extracted
return words_list or []
logger.warning(f"票务识别请求失败 (尝试 {attempt+1}): {response.text[:200]}")
except Exception as e:
logger.warning(f"票务识别异常 (尝试 {attempt+1}): {e}")
if attempt < self.max_retries - 1:
time.sleep(self.retry_delay * (2 ** attempt))
logger.error("票务识别失败")
return None
def get_excel_result(self, request_id_or_result: Union[str, Dict]) -> Optional[bytes]:
"""
app/core/ocr/metadata_extractor.py
+150 -13
View File
@@ -16,6 +16,14 @@ from typing import List, Optional, Tuple
SUPPLIER_KEYWORDS = (
"供货单", "供应商", "供货方", "批发", "酒行", "商行",
"经销", "专卖店", "配送单", "送货单", "采购单", "订单",
"销售单", "出库单", "经营部", "商贸", "有限公司", "发证机构",
)
# 供应商名称中需要剔除的冗余噪声词
SUPPLIER_NOISE_WORDS = (
"标题", "采购单", "销售单", "入库单", "出库单", "送货单",
"单据", "订单", "配送单", "清单", "供货单", "预览", "详情",
"打印", "副本", "记账", "联", "存根",
)
# 总金额关键词（命中后取该行最近一个金额数字）
@@ -24,6 +32,11 @@ AMOUNT_KEYWORDS = (
"合计", "总计", "总计金额",
)
# 日期关键词（用于辅助定位日期）
DATE_KEYWORDS = (
"单据日期", "下单时间", "日期", "时间", "开单日期", "制单日期", "打印时间", "业务日期", "下单日期",
)
# 日期正则：4 种格式
DATE_PATTERNS = [
# 2026年07月17日 / 2026年7月17日
@@ -52,6 +65,16 @@ class OrderMetadata:
return asdict(self)
# 排除关键词：包含这些词的行绝对不是供应商抬头
EXCLUDE_KEYWORDS = (
"购货单位", "客户名称", "收货地址", "联系电话", "经手人",
"地址", "电话", "传真", "邮编", "网址", "开户行", "账号",
"税号", "业务员", "联系人", "单据编号", "流水号", "页码",
"四川省", "成都市", "武侯区", "双流区", "高新区", "金牛区",
"成华区", "锦江区", "龙泉驿区", "青羊区", "新都区", "温江区",
"街道", "社区", "楼", "号", "室", "路", "巷", "段",
)
class OrderMetadataExtractor:
"""单据元信息识别器。"""
@@ -61,20 +84,35 @@ class OrderMetadataExtractor:
# 供应商名最大长度
MAX_SUPPLIER_LEN = 50
def extract(self, ocr_text: str, ocr_rows: Optional[List[List[str]]] = None) -> OrderMetadata:
def extract(self, ocr_text: str, ocr_rows: Optional[List[List[str]]] = None, general_text: Optional[str] = None) -> OrderMetadata:
"""从 OCR 原始文本和/或解析后的二维数组提取三字段。
Args:
ocr_text: OCR 原始字符串全文（带换行）
ocr_rows: 解析后的二维数组（可选），用于 row-level 精确匹配
ocr_text: OCR 原始字符串全文（从表格 OCR 提取）
ocr_rows: 解析后的二维数组（可选）
general_text: 从通用票据识别接口 (/v1/general_ocr) 提取的文本，优先级最高
Returns:
OrderMetadata
"""
text = ocr_text or ''
supplier, raw_supplier = self._extract_supplier(text, ocr_rows)
bill_date = self._extract_bill_date(text)
total_amount = self._extract_total_amount(text, ocr_rows)
# 优先使用通用票据识别的文本进行供应商和日期提取
supplier_source = general_text if general_text else ocr_text
date_source = general_text if general_text else ocr_text
# 金额通常在表格内，优先使用 ocr_text
amount_source = ocr_text or general_text or ''
supplier, raw_supplier = self._extract_supplier(supplier_source, ocr_rows)
bill_date = self._extract_bill_date(date_source)
total_amount = self._extract_total_amount(amount_source, ocr_rows)
# 如果通用票据识别没抓到供应商，尝试用表格 OCR 的文本补位
if not supplier and general_text and ocr_text:
supplier, raw_supplier = self._extract_supplier(ocr_text, ocr_rows)
# 如果通用票据识别没抓到日期，尝试用表格 OCR 的文本补位
if not bill_date and general_text and ocr_text:
bill_date = self._extract_bill_date(ocr_text)
return OrderMetadata(
supplier=supplier,
@@ -95,9 +133,15 @@ class OrderMetadataExtractor:
cleaned = self._clean_supplier_line(line)
if not cleaned:
continue
# 严格排除：购货单位、地址、电话等干扰项
if any(k in cleaned for k in EXCLUDE_KEYWORDS):
continue
for kw in SUPPLIER_KEYWORDS:
if kw in cleaned:
return self._truncate(cleaned), cleaned
final_name = self._final_cleanup_supplier(cleaned)
return self._truncate(final_name), cleaned
# 2) 兜底：取顶部第一个"含中文且无数字行号"且长度 ≥ 4 的非空行
# 但排除"纯日期行"（避免把日期当供应商）
@@ -105,6 +149,11 @@ class OrderMetadataExtractor:
cleaned = self._clean_supplier_line(line)
if not cleaned:
continue
# 兜底也要严格排除地址和电话行
if any(k in cleaned for k in EXCLUDE_KEYWORDS):
continue
if not (re.search(r'[\u4e00-\u9fa5]', cleaned) and len(cleaned) >= 4):
continue
# 排除：纯日期（YYYY-MM-DD / YYYY/MM/DD / YYYY年MM月DD日 / YYYYMMDD）
@@ -117,16 +166,63 @@ class OrderMetadataExtractor:
# 排除：以"单据"开头的行
if re.match(r'^\s*单据[:：]?', cleaned):
continue
return self._truncate(cleaned), cleaned
final_name = self._final_cleanup_supplier(cleaned)
if final_name:
return self._truncate(final_name), cleaned
return '', ''
@staticmethod
def _final_cleanup_supplier(s: str) -> str:
"""最后的供应商名称深度清理：去除“标题”、“采购单”等噪声。"""
if not s:
return ''
# 1. 统一处理全角和常见符号
s = s.replace('：', ':').replace('（', '(').replace('）', ')')
# 2. 去除“标题:”或“名称:”这类前缀
s = re.sub(r'^(标题|名称|供应商|供货方|单位|商户)[:\s]*', '', s)
# 3. 循环去除噪声词
noise_sorted = sorted(SUPPLIER_NOISE_WORDS, key=len, reverse=True)
changed = True
while changed:
original = s
for noise in noise_sorted:
# 简单替换所有匹配的噪声词
s = s.replace(noise, '')
# 去除括号及其中间的噪声词（如 (采购单) ）
s = re.sub(r'\(\s*\)', '', s)
# 去除首尾残留标点和空白
s = re.sub(r'^[:：\s\-_\|\.\(\)]+', '', s)
s = re.sub(r'[:：\s\-_\|\.\(\)]+$', '', s)
changed = (s != original)
return s.strip()
@staticmethod
def _clean_supplier_line(line: str) -> str:
"""清理一行文本：去前后空白、去首尾日期/编号/电话/标点。"""
s = line.strip()
if not s:
return ''
# 统一全角冒号
s = s.replace('：', ':')
# 如果是“标题: 新双利采购单”，保留冒号后面的部分进行初步处理
if ':' in s:
parts = s.split(':', 1)
# 如果冒号前面是“标题”、“名称”等词，则取后面
if any(k in parts[0] for k in ("标题", "名称", "供应商", "商户")):
s = parts[1].strip()
# 去行首日期/编号前缀
s = re.sub(r'^[\s\d\-\.\/年月日:：]+', '', s)
# 去行尾标点
@@ -142,13 +238,54 @@ class OrderMetadataExtractor:
def _extract_bill_date(self, text: str) -> str:
"""提取单据日期，标准化为 YYYYMMDD。"""
if not text:
return ''
lines = text.splitlines()
# 1. 优先尝试关键词定位逻辑（如用户建议：搜索“时间”或“日期”）
for i, line in enumerate(lines):
line_clean = line.strip().replace('：', ':')
# 检查是否包含日期相关关键词
for kw in DATE_KEYWORDS:
if kw in line_clean:
# a) 尝试在当前行找日期
for pat in DATE_PATTERNS:
m = pat.search(line_clean)
if m:
y, mo, d = m.group(1), m.group(2), m.group(3)
if self._is_valid_date(y, mo, d):
return f"{y}{int(mo):02d}{int(d):02d}"
# b) 如果当前行没找到，尝试在下一行找（针对表格 OCR 错位情况）
if i + 1 < len(lines):
next_line = lines[i+1].strip()
for pat in DATE_PATTERNS:
m = pat.search(next_line)
if m:
y, mo, d = m.group(1), m.group(2), m.group(3)
if self._is_valid_date(y, mo, d):
return f"{y}{int(mo):02d}{int(d):02d}"
# 2. 兜底策略：全文正则匹配
for pat in DATE_PATTERNS:
m = pat.search(text)
if m:
# 优先找包含 202 开头的年份（更像当前日期）
matches = pat.finditer(text)
for m in matches:
y, mo, d = m.group(1), m.group(2), m.group(3)
if self._is_valid_date(y, mo, d):
return f"{y}{int(mo):02d}{int(d):02d}"
return ''
# 如果年份以 202 开头，优先返回
if y.startswith('202'):
return f"{y}{int(mo):02d}{int(d):02d}"
# 否则记录下来作为候选
last_valid = f"{y}{int(mo):02d}{int(d):02d}"
# 如果没有 202 开头的，返回最后一个有效的
try:
return last_valid
except NameError:
return ''
@staticmethod
def _is_valid_date(y: str, m: str, d: str) -> bool:
app/core/ocr/table_ocr.py
+11 -7
View File
@@ -10,8 +10,8 @@ import base64
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Callable
from ..utils.log_utils import get_logger
from ..utils.file_utils import (
from app.core.utils.log_utils import get_logger
from app.core.utils.file_utils import (
ensure_dir,
get_file_extension,
get_files_by_extensions,
@@ -20,6 +20,7 @@ from ..utils.file_utils import (
load_json,
save_json
)
from app.config.settings import ConfigManager
from .baidu_ocr import BaiduOCRClient
logger = get_logger(__name__)
@@ -102,14 +103,16 @@ class OCRProcessor:
OCR处理器，负责协调OCR识别和结果处理
"""
def __init__(self, config):
def __init__(self, config: Optional[ConfigManager] = None):
"""
初始化OCR处理器
Args:
config: 配置信息
config: 配置管理器
"""
self.config = config
self.config = config or ConfigManager()
self.ocr_client = None
self._ensure_ocr_client()
# 修复ConfigParser对象没有get_path方法的问题
try:
@@ -348,9 +351,10 @@ class OCRProcessor:
if max_workers is None:
try:
max_workers = self.config.getint('Performance', 'max_workers', fallback=4)
# 强制设为 1 以严格遵守百度 QPS=2 的限制（一张图调两个接口就满了）
max_workers = self.config.getint('Performance', 'max_workers', fallback=1)
except Exception:
max_workers = 4
max_workers = 1
# 获取未处理的图片
unprocessed_images = self.get_unprocessed_images()
app/core/processors/__init__.py
+3 -3
View File
@@ -2,8 +2,8 @@
处理器模块初始化文件
"""
from .base import BaseProcessor
from .ocr_processor import OCRProcessor
from .tobacco_processor import TobaccoProcessor
from app.core.processors.base import BaseProcessor
from app.core.processors.ocr_processor import OCRProcessor
from app.core.processors.tobacco_processor import TobaccoProcessor
__all__ = ['BaseProcessor', 'OCRProcessor', 'TobaccoProcessor']
app/core/processors/base.py
+1 -1
View File
@@ -10,7 +10,7 @@ from pathlib import Path
import logging
import pandas as pd
from ...core.utils.log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/processors/ocr_processor.py
+4 -4
View File
@@ -8,10 +8,10 @@ import os
from pathlib import Path
from typing import Optional, Dict, Any, List
from .base import BaseProcessor
from ...services.ocr_service import OCRService
from ...services.order_service import OrderService
from ...core.utils.log_utils import get_logger
from app.core.processors.base import BaseProcessor
from app.services.ocr_service import OCRService
from app.services.order_service import OrderService
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/processors/supplier_processors/__init__.py
+1 -1
View File
@@ -2,6 +2,6 @@
供应商处理器模块初始化文件
"""
from .generic_supplier_processor import GenericSupplierProcessor
from app.core.processors.supplier_processors.generic_supplier_processor import GenericSupplierProcessor
__all__ = ['GenericSupplierProcessor']
app/core/processors/supplier_processors/generic_supplier_processor.py
+6 -6
View File
@@ -9,12 +9,12 @@ import pandas as pd
from typing import Optional, Dict, Any, List
from pathlib import Path
from ..base import BaseProcessor
from ...utils.log_utils import get_logger
from ...handlers.rule_engine import apply_rules
from ...handlers.column_mapper import ColumnMapper
from ...handlers.data_cleaner import DataCleaner
from ...handlers.calculator import DataCalculator
from app.core.processors.base import BaseProcessor
from app.core.utils.log_utils import get_logger
from app.core.handlers.rule_engine import apply_rules
from app.core.handlers.column_mapper import ColumnMapper
from app.core.handlers.data_cleaner import DataCleaner
from app.core.handlers.calculator import DataCalculator
logger = get_logger(__name__)
app/core/processors/tobacco_processor.py
+4 -4
View File
@@ -14,10 +14,10 @@ from openpyxl import load_workbook
from typing import Optional, Dict, Any, List, Tuple
from pathlib import Path
from .base import BaseProcessor
from ...core.utils.log_utils import get_logger
from ...core.utils.string_utils import parse_monetary_string
from ...core.utils.dialog_utils import show_custom_dialog
from app.core.processors.base import BaseProcessor
from app.core.utils.log_utils import get_logger
from app.core.utils.string_utils import parse_monetary_string
from app.core.utils.dialog_utils import show_custom_dialog
logger = get_logger(__name__)
app/core/utils/cloud_sync.py
+1 -1
View File
@@ -6,7 +6,7 @@ from typing import Optional, Tuple
import requests
from .log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)
app/core/utils/dialog_utils.py
+6 -6
View File
@@ -13,7 +13,7 @@ import tkinter as tk
from tkinter import messagebox, ttk, simpledialog
from datetime import datetime
from .cloud_sync import GiteaSync
from app.core.utils.cloud_sync import GiteaSync
from app.config.settings import ConfigManager
def create_custom_dialog(title="提示", message="", result_file=None, time_info=None,
@@ -82,7 +82,7 @@ def create_custom_dialog(title="提示", message="", result_file=None, time_info
file_size = os.path.getsize(result_file)
file_time = datetime.fromtimestamp(os.path.getmtime(result_file))
from .file_utils import format_file_size
from app.core.utils.file_utils import format_file_size
size_text = format_file_size(file_size)
tk.Label(file_frame, text=f"文件大小: {size_text}", font=("Arial", 10)).pack(anchor=tk.W, padx=10, pady=2)
@@ -831,10 +831,10 @@ SYNC_FILES = [
"type": "binary",
},
{
"name": "商品记忆库",
"remote": "product_memory.json",
"local": "data/product_memory.json",
"type": "json",
"name": "商品记忆库 (DB)",
"remote": "product_cache.db",
"local": "data/product_cache.db",
"type": "binary",
},
]
app/core/utils/file_utils.py
+1 -1
View File
@@ -12,7 +12,7 @@ from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Union, Any
from .log_utils import get_logger
from app.core.utils.log_utils import get_logger
logger = get_logger(__name__)