- 实测订单接口平均 300ms,2.5 次/秒连续 20 次无限流,故营业时段可压到 2 秒 - 商家后台 JS 无 WebSocket/SSE/socket.io/MQTT(0 命中)→ 平台不推,只能轮询 - 出单后 60 秒内自动加密到 1.5 秒(出单常成串),窗口结束回落 - 新增 last_detect_lag_sec:以订单 payTime 计算「支付→检出」真实延迟,写入 state 并暴露在 /healthz - 新增 6 项节奏测试(共 30 项全绿) - 修复 login_url 折行产生空格、上传内网绕过系统代理、新增 --upload-only 补传
483 lines
20 KiB
Python
483 lines
20 KiB
Python
"""监控主循环。
|
||
|
||
与原实现的行为差异(都是有意的):
|
||
1. 首次启动只建立基线,不推送历史订单(原来初始拉单失败会把 20 单全推一遍)
|
||
2. 夜间暂停改为分片休眠:每 60 秒醒一次,可感知 Cookie 更新与停止信号(原来 sleep 10 小时)
|
||
3. Cookie 失效不再弹浏览器(服务器没桌面),改为告警 + 轮询等待新 Cookie,收到即自动恢复
|
||
4. Cookie 文件变化自动热重载,无需重启容器
|
||
5. 网络抖动走指数退避,连续失败才升级告警
|
||
6. 恢复后积压订单超过阈值改发汇总,避免刷屏
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import datetime
|
||
import json
|
||
import logging
|
||
import os
|
||
import random
|
||
import signal
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from .client import ApiError, FengxiangClient, FengxiangError, NetworkError, SessionExpired
|
||
from .config import Config
|
||
from .notify import Dispatcher, build_channels
|
||
from .store import State
|
||
from .tts import Speaker
|
||
|
||
log = logging.getLogger("fengxiang.monitor")
|
||
|
||
COOKIE_WAIT_POLL_SEC = 15 # 等待新 Cookie 时的轮询间隔
|
||
COOKIE_RECHECK_SEC = 300 # 即使文件没变,也每 5 分钟再验一次
|
||
STATE_SAVE_SEC = 600 # 无事件时的状态落盘间隔
|
||
|
||
|
||
def fmt_money(cents: Any) -> str:
|
||
try:
|
||
return f"{int(cents) / 100:.2f}元"
|
||
except (TypeError, ValueError):
|
||
return "0.00元"
|
||
|
||
|
||
def format_order_message(order: Dict[str, Any]) -> str:
|
||
return f"【丰享丰食】订单收款成功,收款{fmt_money(order.get('amountPayable'))}"
|
||
|
||
|
||
class Monitor:
|
||
def __init__(self, cfg: Config, client: Optional[FengxiangClient] = None,
|
||
dispatcher: Optional[Dispatcher] = None,
|
||
speaker: Optional[Speaker] = None):
|
||
self.cfg = cfg
|
||
self.client = client or FengxiangClient(cfg)
|
||
self.dispatcher = dispatcher or Dispatcher(
|
||
build_channels(cfg), cfg.notify.alert_cooldown_sec
|
||
)
|
||
self.speaker = speaker or Speaker(cfg.tts.voice_prompt, cfg.tts.enabled)
|
||
self.state = State.load(cfg.paths.state)
|
||
self.stop_event = threading.Event()
|
||
self._cookie_sig: Optional[tuple] = None
|
||
self._last_state_save = 0.0
|
||
self._net_failures = 0
|
||
self._burst_until = 0.0 # 出单后的高频追击截止时刻
|
||
self._started_at = time.time()
|
||
self._admin = None
|
||
self._last_cookie_check = 0.0
|
||
self._now_impl = self._default_now
|
||
|
||
# ------------------------------------------------------------------ 时间
|
||
def now(self) -> datetime.datetime:
|
||
"""当前时间。测试可通过替换 _now_impl 注入固定时间。"""
|
||
return self._now_impl()
|
||
|
||
def _default_now(self) -> datetime.datetime:
|
||
try:
|
||
import zoneinfo
|
||
return datetime.datetime.now(zoneinfo.ZoneInfo(self.cfg.timezone))
|
||
except Exception: # noqa: BLE001 - 无 tzdata 时退回本机时区
|
||
return datetime.datetime.now()
|
||
|
||
def is_peak(self) -> bool:
|
||
hour = self.now().hour
|
||
return any(start <= hour < end for start, end in self.cfg.polling.peak_hours)
|
||
|
||
def is_business(self) -> bool:
|
||
"""是否处于营业时段(决定轮询用 2 秒还是 8 秒)。"""
|
||
hour = self.now().hour
|
||
return any(start <= hour < end for start, end in self.cfg.polling.business_hours)
|
||
|
||
def is_night(self) -> bool:
|
||
n = self.now()
|
||
minutes = n.hour * 60 + n.minute
|
||
p = self.cfg.polling
|
||
return minutes >= p.night_start_min or minutes < p.night_end_min
|
||
|
||
def seconds_until_morning(self) -> int:
|
||
n = self.now()
|
||
target = n.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(
|
||
minutes=self.cfg.polling.night_end_min
|
||
)
|
||
if target <= n:
|
||
target += datetime.timedelta(days=1)
|
||
return max(1, int((target - n).total_seconds()))
|
||
|
||
def poll_page_size(self) -> int:
|
||
p = self.cfg.polling
|
||
return p.peak_page_size if self.is_peak() else p.idle_page_size
|
||
|
||
# ------------------------------------------------------------------ Cookie
|
||
def _cookie_sig_of_file(self) -> Optional[tuple]:
|
||
path = Path(self.cfg.paths.cookies)
|
||
if not path.exists():
|
||
return None
|
||
st = path.stat()
|
||
return (st.st_mtime_ns, st.st_size)
|
||
|
||
def read_cookies(self) -> Optional[Dict[str, str]]:
|
||
path = Path(self.cfg.paths.cookies)
|
||
if not path.exists():
|
||
return None
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
if isinstance(data, dict) and data:
|
||
return {str(k): str(v) for k, v in data.items()}
|
||
log.warning("cookie 文件内容不是非空字典,忽略")
|
||
return None
|
||
except (OSError, ValueError) as exc:
|
||
log.error("读取 cookie 文件失败: %s", exc)
|
||
return None
|
||
|
||
def apply_cookies(self, cookies: Dict[str, str], source: str = "file",
|
||
validated: bool = False) -> bool:
|
||
"""应用 Cookie(供主循环与管理接口共用)。
|
||
|
||
validated=True 表示调用方已经校验过,跳过重复的网络校验。
|
||
"""
|
||
if not cookies:
|
||
return False
|
||
if not validated and not self.client.check_cookie(cookies):
|
||
log.warning("来自 %s 的 Cookie 校验未通过", source)
|
||
return False
|
||
self.client.set_cookies(cookies)
|
||
old = self.state.cookie_updated_ts
|
||
self.state.cookie_updated_ts = time.time()
|
||
self.state.cookie_valid = True
|
||
self.state.save()
|
||
log.info("Cookie 已生效(来源=%s,共 %d 项,距上次更新 %.0f 秒)",
|
||
source, len(cookies), self.state.cookie_updated_ts - old if old else 0)
|
||
if old and self.state.gap_since_ts:
|
||
gap_min = (time.time() - self.state.gap_since_ts) / 60
|
||
self.dispatcher.notify(f"【丰享监控】已重新登录,监控恢复(中断约 {gap_min:.0f} 分钟)")
|
||
self.state.gap_since_ts = 0.0
|
||
self.state.save()
|
||
return True
|
||
|
||
def _load_cookie_if_changed(self, force: bool = False) -> bool:
|
||
sig = self._cookie_sig_of_file()
|
||
if sig is None:
|
||
if force:
|
||
log.warning("未找到 Cookie 文件: %s", self.cfg.paths.cookies)
|
||
return False
|
||
if not force and sig == self._cookie_sig:
|
||
return True
|
||
cookies = self.read_cookies()
|
||
ok = self.apply_cookies(cookies or {}, source="file")
|
||
if ok:
|
||
self._cookie_sig = sig
|
||
else:
|
||
# 记录签名,避免同一份坏 Cookie 反复校验
|
||
self._cookie_sig = sig
|
||
return ok
|
||
|
||
def update_cookies(self, cookies: Dict[str, str]) -> bool:
|
||
"""管理接口收到新 Cookie:先校验再落盘,避免坏 Cookie 覆盖掉正在用的好 Cookie。"""
|
||
from .store import atomic_write_json
|
||
|
||
if not cookies:
|
||
return False
|
||
if not self.client.check_cookie(cookies):
|
||
log.warning("管理接口提交的 Cookie 校验未通过,已拒绝(保留原有 Cookie)")
|
||
return False
|
||
atomic_write_json(Path(self.cfg.paths.cookies), cookies)
|
||
self._cookie_sig = self._cookie_sig_of_file()
|
||
return self.apply_cookies(cookies, source="admin-api", validated=True)
|
||
|
||
# ------------------------------------------------------------------ 状态
|
||
def status(self) -> Dict[str, Any]:
|
||
now = time.time()
|
||
return {
|
||
"app": "fengxiang-monitor",
|
||
"mode": ("night" if self.is_night() else ("peak" if self.is_peak() else "idle")),
|
||
"cookie_valid": bool(self.state.cookie_valid),
|
||
"cookie_age_sec": int(now - self.state.cookie_updated_ts) if self.state.cookie_updated_ts else None,
|
||
"last_order_id": self.state.last_order_id,
|
||
"last_poll_age_sec": int(now - self.state.last_poll_ts) if self.state.last_poll_ts else None,
|
||
"total_pushed": self.state.total_pushed,
|
||
"uptime_sec": int(now - self._started_at),
|
||
"net_failures": self._net_failures,
|
||
"last_detect_lag_sec": self.state.last_detect_lag_sec,
|
||
"polling_interval_sec": round(self._next_interval(), 2),
|
||
"bursting": time.time() < self._burst_until,
|
||
}
|
||
|
||
def _save_state(self, force: bool = False) -> None:
|
||
now = time.time()
|
||
if force or now - self._last_state_save > STATE_SAVE_SEC:
|
||
self.state.save()
|
||
self._last_state_save = now
|
||
|
||
# ------------------------------------------------------------------ 主流程
|
||
def stop(self, *_args) -> None:
|
||
log.info("收到停止信号,准备退出")
|
||
self.stop_event.set()
|
||
|
||
def _install_signals(self) -> None:
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
try:
|
||
signal.signal(sig, self.stop)
|
||
except (ValueError, OSError): # 非主线程/平台不支持
|
||
pass
|
||
|
||
def run(self) -> None:
|
||
self._install_signals()
|
||
self.speaker.start()
|
||
log.info("丰享订单监控启动 v%s | shop=%s | 通道=%s",
|
||
__import__("fengxiang").__version__, self.cfg.shop_id,
|
||
",".join(c.name for c in self.dispatcher.channels) or "无")
|
||
|
||
if self.cfg.admin.enabled:
|
||
from .admin import AdminServer
|
||
self._admin = AdminServer(self.cfg, self)
|
||
self._admin.start()
|
||
|
||
# 1) 确保有可用 Cookie
|
||
if not self._ensure_cookie():
|
||
log.error("启动阶段未获得可用 Cookie,进入等待模式")
|
||
if self.stop_event.is_set():
|
||
self._shutdown()
|
||
return
|
||
|
||
# 2) 建立基线(不推送历史订单)
|
||
if self.state.cookie_valid:
|
||
self._bootstrap()
|
||
|
||
# 3) 主循环
|
||
while not self.stop_event.is_set():
|
||
try:
|
||
if self.is_night():
|
||
self._night_wait()
|
||
continue
|
||
|
||
self._load_cookie_if_changed(force=False)
|
||
|
||
page_size = self.poll_page_size()
|
||
try:
|
||
orders = self.client.list_orders(page_size=page_size)
|
||
except SessionExpired as exc:
|
||
self._on_session_expired(str(exc))
|
||
continue
|
||
except (NetworkError, ApiError) as exc:
|
||
self._on_soft_error(exc)
|
||
continue
|
||
|
||
self._net_failures = 0
|
||
self.state.touch_poll()
|
||
self._process(orders, page_size)
|
||
self._save_state()
|
||
self._wait(self._next_interval())
|
||
except Exception as exc: # noqa: BLE001 - 主循环绝不能被单次异常打死
|
||
log.exception("主循环异常(已忽略并继续): %s", exc)
|
||
self._wait(10)
|
||
|
||
self._shutdown()
|
||
|
||
def _shutdown(self) -> None:
|
||
self.speaker.stop()
|
||
if self._admin:
|
||
self._admin.stop()
|
||
self._save_state(force=True)
|
||
log.info("监控已停止")
|
||
|
||
def _ensure_cookie(self) -> bool:
|
||
"""启动时确保 Cookie 可用;不可用则等待(不弹浏览器)。"""
|
||
if self._load_cookie_if_changed(force=True):
|
||
log.info("启动校验通过,复用已有 Cookie")
|
||
return True
|
||
|
||
cookies = self.read_cookies()
|
||
if cookies and self.client.check_cookie(cookies):
|
||
return self.apply_cookies(cookies, source="file")
|
||
|
||
self.state.cookie_valid = False
|
||
self.state.gap_since_ts = self.state.gap_since_ts or time.time()
|
||
self.state.save()
|
||
self.dispatcher.alert(
|
||
"cookie_need_login",
|
||
"【丰享监控】需要登录:Cookie 缺失或已失效。请在本机运行 fengxiang_scraper.py 完成登录,"
|
||
"脚本会自动上传 Cookie,服务器收到后自动恢复监控。",
|
||
)
|
||
return self._wait_for_new_cookie()
|
||
|
||
def _wait_for_new_cookie(self) -> bool:
|
||
"""等待新 Cookie 落地。返回是否拿到可用 Cookie。"""
|
||
baseline = self._cookie_sig_of_file()
|
||
log.warning("进入 Cookie 等待模式(等待 %s 更新)", self.cfg.paths.cookies)
|
||
last_log = 0.0
|
||
last_recheck = 0.0
|
||
while not self.stop_event.is_set():
|
||
self.stop_event.wait(COOKIE_WAIT_POLL_SEC)
|
||
if self.stop_event.is_set():
|
||
return False
|
||
|
||
sig = self._cookie_sig_of_file()
|
||
if sig and sig != baseline:
|
||
cookies = self.read_cookies()
|
||
if cookies and self.apply_cookies(cookies, source="file"):
|
||
self._cookie_sig = sig
|
||
return True
|
||
baseline = sig
|
||
log.warning("新 Cookie 仍不可用,继续等待")
|
||
|
||
# 兜底:文件没变化但可能是同一份 Cookie 恢复了(服务端续期)
|
||
now = time.time()
|
||
if now - last_recheck > COOKIE_RECHECK_SEC:
|
||
last_recheck = now
|
||
cookies = self.read_cookies()
|
||
if cookies and self.client.check_cookie(cookies):
|
||
if self.apply_cookies(cookies, source="recheck"):
|
||
return True
|
||
if now - last_log > 300:
|
||
last_log = now
|
||
log.warning("仍在等待 Cookie(已等待 %.0f 分钟)", (now - (self.state.gap_since_ts or now)) / 60)
|
||
return False
|
||
|
||
def _on_session_expired(self, reason: str) -> None:
|
||
log.error("会话失效: %s", reason)
|
||
self.state.cookie_valid = False
|
||
self.state.gap_since_ts = self.state.gap_since_ts or time.time()
|
||
self.state.save()
|
||
self.dispatcher.alert(
|
||
"cookie_expired",
|
||
"【丰享监控】登录已失效(Cookie 过期),监控暂停。"
|
||
"请在本机运行 fengxiang_scraper.py 重新登录,上传后自动恢复。",
|
||
)
|
||
self._wait_for_new_cookie()
|
||
|
||
def _on_soft_error(self, exc: FengxiangError) -> None:
|
||
self._net_failures += 1
|
||
backoff = min(60, 5 * (2 ** min(self._net_failures - 1, 4))) + random.uniform(0, 3)
|
||
log.warning("拉取订单失败(第 %d 次): %s,%.1f 秒后重试",
|
||
self._net_failures, exc, backoff)
|
||
if self._net_failures in (5, 20, 60):
|
||
self.dispatcher.alert(
|
||
f"net_fail_{self._net_failures}",
|
||
f"【丰享监控】连续 {self._net_failures} 次拉单失败,最近错误:{exc}",
|
||
)
|
||
self._wait(backoff)
|
||
|
||
def _bootstrap(self) -> None:
|
||
"""建立订单基线:取最新一单作为参考点,不推送。"""
|
||
if self.state.last_order_id:
|
||
return
|
||
try:
|
||
orders = self.client.list_orders(page_size=1)
|
||
except FengxiangError as exc:
|
||
log.warning("建立基线失败(下次轮询再试): %s", exc)
|
||
return
|
||
if orders:
|
||
self.state.last_order_id = str(orders[0].get("orderId"))
|
||
self.state.mark_pushed([self.state.last_order_id])
|
||
self.state.save()
|
||
log.info("已建立基线,当前最新订单 %s(不推送历史订单)", self.state.last_order_id)
|
||
|
||
def _process(self, orders: List[Dict[str, Any]], page_size: int) -> None:
|
||
if not orders:
|
||
log.debug("本次拉取为空")
|
||
return
|
||
|
||
new_orders: List[Dict[str, Any]] = []
|
||
for order in orders:
|
||
oid = str(order.get("orderId") or "")
|
||
if not oid:
|
||
continue
|
||
if self.state.last_order_id and oid == self.state.last_order_id:
|
||
break
|
||
if self.state.has_pushed(oid):
|
||
continue
|
||
new_orders.append(order)
|
||
|
||
if not new_orders:
|
||
log.debug("暂无新订单(最新=%s)", self.state.last_order_id)
|
||
return
|
||
|
||
if len(new_orders) >= page_size:
|
||
log.warning("本次新订单数达到页大小 %d,可能仍有更早订单未拉取,建议临时调大 page_size",
|
||
page_size)
|
||
|
||
newest_id = str(new_orders[0].get("orderId"))
|
||
threshold = self.cfg.polling.bulk_summary_threshold
|
||
|
||
if len(new_orders) > threshold:
|
||
total = sum(int(o.get("amountPayable") or 0) for o in new_orders)
|
||
msg = (f"【丰享丰食】监控恢复,期间新增 {len(new_orders)} 笔订单,"
|
||
f"合计 {fmt_money(total)}(已合并提醒)")
|
||
log.info("积压 %d 单,发送汇总消息", len(new_orders))
|
||
self.dispatcher.notify(msg)
|
||
else:
|
||
log.info("发现 %d 笔新订单", len(new_orders))
|
||
for order in reversed(new_orders): # 从旧到新
|
||
msg = format_order_message(order)
|
||
self.dispatcher.notify(msg)
|
||
self.speaker.say(msg)
|
||
time.sleep(0.3)
|
||
|
||
# 刚出一单,接下来一小段时间加密轮询(出单常成串)
|
||
self._burst_until = time.time() + self.cfg.polling.burst_window_sec
|
||
lag = self._detect_lag(new_orders[0])
|
||
if lag is not None:
|
||
self.state.last_detect_lag_sec = round(lag, 1)
|
||
log.info("本单端到端延迟 %.1f 秒(支付→检出),此后 %d 秒内按 %.1f 秒追击",
|
||
lag, self.cfg.polling.burst_window_sec, self.cfg.polling.interval_burst)
|
||
|
||
self.state.mark_pushed([str(o.get("orderId")) for o in new_orders])
|
||
self.state.last_order_id = newest_id
|
||
self.state.save()
|
||
|
||
def _detect_lag(self, order: Dict[str, Any]) -> Optional[float]:
|
||
"""端到端延迟 = 现在 - 支付时间。
|
||
|
||
这是唯一能证明「到底多快」的指标:它同时包含平台落单延迟 + 我们的轮询间隔。
|
||
超过 24 小时的视为脏数据(时区/补推历史单)直接丢弃。
|
||
"""
|
||
try:
|
||
pay = float(order.get("payTime") or order.get("createTime") or 0)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if pay <= 0:
|
||
return None
|
||
lag = time.time() - pay
|
||
return lag if 0 <= lag < 86400 else None
|
||
|
||
def _next_interval(self) -> float:
|
||
"""分时段间隔:刚出单后加密,营业时段快,非营业时段慢。
|
||
|
||
实测订单接口平均 300ms、2.5 次/秒无任何限流,因此营业时段可以压到 2 秒;
|
||
刚推完一单后 60 秒内用 1.5 秒(出单常成串),窗口过了自动回落。
|
||
"""
|
||
p = self.cfg.polling
|
||
if time.time() < self._burst_until:
|
||
base = p.interval_burst
|
||
elif self.is_business():
|
||
base = p.interval_business
|
||
else:
|
||
base = p.interval_idle
|
||
return max(0.5, base + random.uniform(-0.2, 0.2))
|
||
|
||
def _wait(self, seconds: float) -> None:
|
||
"""可中断等待。"""
|
||
if seconds > 0:
|
||
self.stop_event.wait(seconds)
|
||
|
||
def _night_wait(self) -> None:
|
||
remaining = self.seconds_until_morning()
|
||
step = min(remaining, self.cfg.polling.night_check_sec)
|
||
if remaining == step:
|
||
log.info("夜间暂停,%.0f 分钟后恢复", remaining / 60)
|
||
# 分片休眠:便于感知 Cookie 更新和停止信号
|
||
waited = 0.0
|
||
while waited < remaining and not self.stop_event.is_set():
|
||
chunk = min(self.cfg.polling.night_check_sec, remaining - waited)
|
||
self.stop_event.wait(chunk)
|
||
waited += chunk
|
||
self._load_cookie_if_changed(force=False)
|
||
if not self.stop_event.is_set():
|
||
log.info("夜间结束,恢复监控")
|
||
self._bootstrap()
|
||
|
||
|
||
def build_monitor(cfg: Config) -> Monitor:
|
||
return Monitor(cfg)
|