Update: Refactor to absolute imports, fix QPS limits, and enhance Gitea sync with SQLite support

2026-07-21 10:25:29 +08:00
parent ef04fc5627
commit 97d9d98b4c
67 changed files with 1691 additions and 457 deletions
@@ -12,13 +12,13 @@ from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union, Any, Callable

-from ..config.settings import ConfigManager
-from ..core.utils.log_utils import get_logger
-from ..core.excel.processor import ExcelProcessor
-from ..core.excel.merger import PurchaseOrderMerger
-from ..core.db.product_db import ProductDatabase
-from ..core.db.order_metadata_db import OrderMetadataDB
-from ..core.ocr.metadata_extractor import OrderMetadataExtractor, sanitize_for_filename
+from app.config.settings import ConfigManager
+from app.core.utils.log_utils import get_logger
+from app.core.excel.processor import ExcelProcessor
+from app.core.excel.merger import PurchaseOrderMerger
+from app.core.db.product_db import ProductDatabase
+from app.core.db.order_metadata_db import OrderMetadataDB
+from app.core.ocr.metadata_extractor import OrderMetadataExtractor, sanitize_for_filename

 logger = get_logger(__name__)

@@ -27,15 +27,17 @@ class OrderService:
    订单服务：协调Excel处理和订单合并流程
    """
    
-    def __init__(self, config: Optional[ConfigManager] = None):
+    def __init__(self, config: Optional[ConfigManager] = None, missing_barcodes_cb: Optional[Callable[[List[str]], None]] = None):
        """
        初始化订单服务
        
        Args:
            config: 配置管理器，如果为None则创建新的
+            missing_barcodes_cb: 缺失条码的回调函数
        """
        logger.info("初始化OrderService")
        self.config = config or ConfigManager()
+        self.missing_barcodes_cb = missing_barcodes_cb

        # 创建共享的商品数据库实例
        db_path = self.config.get_path('Paths', 'product_db', fallback='data/product_cache.db') if hasattr(self.config, 'get_path') else 'data/product_cache.db'
@@ -45,7 +47,11 @@ class OrderService:
        self.product_db = ProductDatabase(db_path, tpl_path)

        # 创建Excel处理器和采购单合并器
-        self.excel_processor = ExcelProcessor(self.config, product_db=self.product_db)
+        self.excel_processor = ExcelProcessor(
+            self.config, 
+            product_db=self.product_db,
+            missing_barcodes_cb=self.missing_barcodes_cb
+        )
        self.order_merger = PurchaseOrderMerger(self.config)

        # 元信息识别器 + 单据元信息库
@@ -291,15 +297,16 @@ class OrderService:
            if meta_path.exists():
                try:
                    payload = json.loads(meta_path.read_text(encoding='utf-8'))
-                    # 优先用通用识别文本（含手写抬头/日期）
-                    ocr_text = payload.get('general_text') or payload.get('ocr_text', '') or ''
+                    # 区分表格 OCR 文本和通用识别文本
+                    ocr_text = payload.get('ocr_text', '') or ''
+                    general_text = payload.get('general_text', '') or ''
                    ocr_rows = payload.get('ocr_rows', []) or []
                    source_image = payload.get('image_path', '') or ''
                except Exception as e:
                    logger.warning(f"读 meta.json 失败: {e}")

            # 兜底：从 xlsx 拼文本（与 OCRService._write_meta_json 的兜底一致）
-            if not ocr_text:
+            if not ocr_text and not general_text:
                try:
                    import xlrd
                    rb = xlrd.open_workbook(str(ocr_excel_path))
@@ -313,7 +320,7 @@ class OrderService:
                except Exception as e:
                    logger.debug(f"从 xlsx 拼 OCR 文本失败: {e}")

-            meta = self.extractor.extract(ocr_text, ocr_rows)
+            meta = self.extractor.extract(ocr_text, ocr_rows, general_text=general_text)
            self.metadata_db.save(
                file_hash=file_hash,
                supplier=meta.supplier,
@@ -334,49 +341,77 @@ class OrderService:
    def _apply_metadata_to_filenames(self, result_path: str,
                                      ocr_excel_path: str,
                                      meta) -> str:
-        """应用新文件名规则：
-        - result: 采购单_{YYYYMMDD}_{供应商}_{hash}.xls
-        - 原图: {原stem}_{YYYYMMDD}_{供应商}_{hash}.{ext}
+        """应用新文件名规则（按供应商+日期）：
+        - result (xls)：{供应商}_{日期}.xls
+        - output xlsx：  {供应商}_{日期}.xlsx
+        - 原图：       {供应商}_{日期}.{ext}
+
+        冲突时加 _2 / _3 ...；任一步骤失败不影响整体。

-        任一步骤失败不影响 result 文件本身。
        Returns:
            新 result 路径（无论重命名是否成功都返回；失败时返回原路径）
        """
        try:
-            file_hash = Path(ocr_excel_path).stem
+            # 1. 提取并清理元数据
            supplier_clean = sanitize_for_filename(meta.supplier) or '未知供应商'
            date_part = meta.bill_date or '未知日期'

-            # ── 1. result 重命名 ──
-            new_result_name = f"采购单_{date_part}_{supplier_clean}_{file_hash}.xls"
-            result_dir = Path(result_path).parent
-            new_result_path = result_dir / new_result_name
-            try:
-                # 冲突时加 _N
-                if new_result_path.exists() and str(new_result_path) != str(result_path):
-                    new_result_path = self._dedup_path(new_result_path)
-                if str(new_result_path) != str(result_path):
-                    os.rename(result_path, str(new_result_path))
-                    logger.info(f"result 重命名: {result_path} -> {new_result_path}")
-                    result_path = str(new_result_path)
-            except Exception as e:
-                logger.warning(f"result 重命名失败: {e}")
+            # 2. 构造新的基础文件名：[供应商]_[日期]
+            # 按照用户要求：按照供应商名称加日期进行修改，且不含“采购单”等冗余字眼
+            base_name = f"{supplier_clean}_{date_part}"

-            # ── 2. 原图重命名 ──
+            file_hash = Path(ocr_excel_path).stem  # 仅用作 SQLite PK，不入文件名
+            logger.info(f"生成标准化文件名: {base_name}")
+
+            # ── 1. result 重命名 ──
+            new_result_path = self._safe_rename(
+                result_path, f"{base_name}.xls"
+            )
+
+            # ── 2. output xlsx 重命名（保证 output/ 与 result/ 文件名一致）──
+            new_output_path = self._safe_rename(
+                ocr_excel_path, f"{base_name}.xlsx"
+            )
+            # 同步 meta.json
+            old_meta = Path(ocr_excel_path).with_suffix('.meta.json')
+            if old_meta.exists():
+                new_meta = Path(new_output_path).with_suffix('.meta.json')
+                if str(new_meta) != str(old_meta):
+                    try:
+                        if new_meta.exists(): os.remove(new_meta)
+                        os.rename(old_meta, new_meta)
+                    except Exception as e:
+                        logger.warning(f"meta.json 重命名失败: {e}")
+
+            # ── 2.5 同步 processed_files.json（避免下次 OCR 重新识别）──
+            if str(new_output_path) != str(ocr_excel_path):
+                self._sync_processed_record(Path(ocr_excel_path), Path(new_output_path))
+                # 也要同步 ExcelProcessor 的记录
+                self._update_excel_process_record(Path(ocr_excel_path), Path(new_output_path))
+
+            # ── 3. 原图重命名 ──
            try:
-                meta_row = self.metadata_db.get(file_hash)
-                src_image = (meta_row or {}).get('source_image', '')
+                # 尝试从 processed_files.json 反查原图路径
+                src_image = self._find_input_image_for_output(str(ocr_excel_path))
+                
+                # 如果没找到，再从 DB 拿
+                if not src_image:
+                    meta_row = self.metadata_db.get(file_hash)
+                    src_image = (meta_row or {}).get('source_image', '')
+                
                if src_image and os.path.exists(src_image):
                    src_p = Path(src_image)
-                    stem = src_p.stem
                    ext = src_p.suffix
-                    new_image_name = f"{stem}_{date_part}_{supplier_clean}_{file_hash}{ext}"
-                    new_image_path = src_p.parent / new_image_name
-                    # 不覆盖已重命名的图片
-                    if str(new_image_path) != str(src_p) and not new_image_path.exists():
-                        os.rename(src_p, new_image_path)
-                        logger.info(f"原图重命名: {src_p.name} -> {new_image_path.name}")
-                        # 更新 source_image 路径
+                    # 严禁包含原文件名，统一格式: 采购单_YYYYMMDD_供应商.ext
+                    new_image_name = f"{base_name}{ext}"
+                    new_image_path = self._safe_rename(src_image, new_image_name)
+                    
+                    # ── 3.5 同步 processed_files.json 的 Key (原图路径) ──
+                    if str(new_image_path) != str(src_p):
+                        self._sync_processed_key(src_p, Path(new_image_path))
+                        
+                    # 更新 source_image 路径到 DB
+                    if str(new_image_path) != str(src_p):
                        self.metadata_db.save(
                            file_hash=file_hash,
                            supplier=meta.supplier,
@@ -385,14 +420,47 @@ class OrderService:
                            raw_supplier_text=meta.raw_supplier_text,
                            source_image=str(new_image_path),
                        )
+                else:
+                    logger.warning(f"找不到原图，跳过重命名: {src_image}")
            except Exception as e:
                logger.warning(f"原图重命名失败: {e}")

-            return result_path
+            return str(new_result_path)
        except Exception as e:
            logger.error(f"_apply_metadata_to_filenames 失败: {e}")
            return result_path

+    @staticmethod
+    def _safe_rename(src: str, new_basename: str) -> str:
+        """把 src 重命名为 src.parent / new_basename。
+
+        - 目标已存在时加 _2 / _3 ... 后缀避免覆盖
+        - 失败返回原路径，不抛异常
+        """
+        try:
+            src_p = Path(src)
+            if not src_p.exists():
+                return src
+            new_p = src_p.parent / new_basename
+            if str(new_p) == str(src_p):
+                return src
+            if new_p.exists():
+                # 冲突去重
+                stem, suffix = new_p.stem, new_p.suffix
+                n = 2
+                while True:
+                    cand = src_p.parent / f"{stem}_{n}{suffix}"
+                    if not cand.exists():
+                        new_p = cand
+                        break
+                    n += 1
+            os.rename(src_p, new_p)
+            logger.info(f"重命名: {src_p.name} -> {new_p.name}")
+            return str(new_p)
+        except Exception as e:
+            logger.warning(f"重命名失败 {src} -> {new_basename}: {e}")
+            return src
+
    @staticmethod
    def _dedup_path(p: Path) -> Path:
        """路径冲突时加 _N 后缀。"""
@@ -404,3 +472,113 @@ class OrderService:
            if not cand.exists():
                return cand
            n += 1
+
+    def _find_input_image_for_output(self, output_xlsx: str) -> Optional[str]:
+        """从 processed_files.json 反查生成该 output_xlsx 的原图路径。"""
+        try:
+            output_dir = self.config.get_path('Paths', 'output_folder', fallback='data/output')
+            record_file = os.path.join(output_dir, 'processed_files.json')
+            
+            if not os.path.exists(record_file):
+                logger.debug(f"记录文件不存在: {record_file}")
+                return None
+                
+            with open(record_file, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            target_output = os.path.normpath(output_xlsx)
+            found_img = None
+            
+            # 1. 尝试直接匹配和规范化路径匹配
+            for img_path, out_path in data.items():
+                if os.path.normpath(out_path) == target_output:
+                    found_img = img_path
+                    break
+            
+            if found_img:
+                if os.path.exists(found_img):
+                    return found_img
+                
+                # 2. 模糊匹配：如果记录中的路径不存在，尝试在同目录下找 stem 匹配的文件
+                # (解决因之前重命名导致的路径不一致问题)
+                img_p = Path(found_img)
+                parent = img_p.parent
+                if parent.exists():
+                    stem = img_p.stem
+                    logger.debug(f"尝试模糊匹配原图: stem={stem[:20]}... in {parent}")
+                    # 尝试寻找以原 stem 开头的文件
+                    for cand in parent.iterdir():
+                        if cand.is_file() and cand.stem.startswith(stem[:20]): 
+                            logger.info(f"模糊匹配成功: {cand.name}")
+                            return str(cand)
+                    logger.debug("模糊匹配失败")
+            else:
+                logger.debug(f"在 processed_files.json 中未找到输出文件 {output_xlsx} 对应的原图记录")
+        except Exception as e:
+            logger.debug(f"反查原图路径失败: {e}")
+        return None
+
+    def _sync_processed_record(self, old_output: Path, new_output: Path) -> None:
+        """重命名 output xlsx 后，把 processed_files.json 里所有 value 为旧路径的项改成新路径。"""
+        self._update_processed_json(old_path=old_output, new_path=new_output, is_key=False)
+
+    def _sync_processed_key(self, old_input: Path, new_input: Path) -> None:
+        """重命名原图后，把 processed_files.json 里的 key 从旧路径改为新路径。"""
+        self._update_processed_json(old_path=old_input, new_path=new_input, is_key=True)
+
+    def _update_excel_process_record(self, old_xlsx: Path, new_xlsx: Path) -> None:
+        """重命名 xlsx 后，同步 excel_process_records.json (ExcelProcessor 使用的记录)。"""
+        self._update_processed_json(
+            old_path=old_xlsx, 
+            new_path=new_xlsx, 
+            is_key=True, 
+            filename='excel_process_records.json'
+        )
+
+    def _update_processed_json(self, old_path: Path, new_path: Path, is_key: bool = False, filename: str = 'processed_files.json') -> None:
+        """更新处理记录 JSON。
+        
+        Args:
+            old_path: 旧路径
+            new_path: 新路径
+            is_key: True 更新 key, False 更新 value
+            filename: JSON 文件名
+        """
+        try:
+            # 确定文件路径
+            output_dir = self.config.get_path('Paths', 'output_folder', fallback='data/output')
+            record_file = os.path.join(output_dir, filename)
+            
+            if not os.path.exists(record_file):
+                return
+                
+            with open(record_file, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            old_str = str(old_path)
+            new_str = str(new_path)
+            changed = False
+            
+            # 规范化路径以便匹配
+            old_norm = os.path.normpath(old_str)
+            
+            if is_key:
+                # 更新 Key
+                for k in list(data.keys()):
+                    if k == old_str or os.path.normpath(k) == old_norm:
+                        data[new_str] = data.pop(k)
+                        changed = True
+                        break
+            else:
+                # 更新 Value
+                for k, v in list(data.items()):
+                    if v == old_str or os.path.normpath(v) == old_norm:
+                        data[k] = new_str
+                        changed = True
+            
+            if changed:
+                with open(record_file, 'w', encoding='utf-8') as f:
+                    json.dump(data, f, ensure_ascii=False, indent=2)
+                logger.info(f"{filename} 已同步 ({'Key' if is_key else 'Value'}): {old_str} -> {new_str}")
+        except Exception as e:
+            logger.warning(f"更新 {filename} 失败: {e}")