refactor: 重构为可部署到服务器的模块化结构(v5.0.0)
- 拆包 fengxiang/:config/client/store/notify/monitor/admin/tts,统一入口 order_monitor.py - 服务器形态:Docker 常驻,Cookie 由本机登录后 POST 上传并热加载,不再依赖桌面弹窗 - 修复:首次启动重推历史订单、speak() 阻塞主循环、夜间 sleep 10h 不可中断 - 修复:test_push.py 残留 miservice 依赖(小爱已移除)、run_monitor.bat 写死盘符 - 配置化:webhook/shopId 移出源码(环境变量 > config.yaml > 默认值) - 可观测:滚动文件日志 + /healthz + Docker HEALTHCHECK + 告警冷却 - 飞书通道保留配置位,默认关闭(原 token 已失效 code=19001) - 新增 22 项单元测试(不联网)
This commit is contained in:
1 parent
ab6a2bb7ea
commit
4ee30399bf
30 files changed
+2410
-825
No files matched your search
+109
-76
@@ -1,114 +1,147 @@
|
||||
"""
|
||||
丰享商家端数据抓取脚本
|
||||
使用 Playwright 半自动登录 + requests 调用数据接口
|
||||
"""本机登录丰享商家端,拿到 Cookie 并(可选)上传到服务器。
|
||||
|
||||
为什么必须在本机跑:丰享登录有腾讯 TCaptcha 滑块/点选,无法纯接口绕过,
|
||||
服务器上又是无桌面的容器,所以把「人工过验证码」这一步留在本机。
|
||||
|
||||
流程:
|
||||
1. Playwright 打开登录页(有头模式),你手动输入账号密码 + 过验证码
|
||||
2. 检测到跳转 MMS 后提取 Cookie → 保存到 ./data/cookies.json
|
||||
3. --upload 时 POST 到服务器管理接口,监控进程立即热加载,无需重启
|
||||
|
||||
用法:
|
||||
python fengxiang_scraper.py # 只登录并保存
|
||||
python fengxiang_scraper.py --upload # 登录 + 上传(地址取环境变量)
|
||||
python fengxiang_scraper.py --upload --server http://192.168.8.210:52100 --token xxx
|
||||
|
||||
环境变量:FX_SERVER / FX_ADMIN_TOKEN / FX_CONFIG
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
# ============ 配置 ============
|
||||
LOGIN_URL = "https://fspass.szfx.com/ucenter/userlogin?platform=saas&redirect_url=https%3A%2F%2Ffs.szfx.com%2FMMS%23%2F&return_url=https%3A%2F%2Ffs.szfx.com%2Fsaasmerchant%2Fsetstoken"
|
||||
DATA_API = "https://fs.szfx.com/saasmerchant/pcweb/order/quickpayorder/list"
|
||||
from fengxiang.config import load_config
|
||||
|
||||
MMS_BASE = "https://fs.szfx.com/MMS"
|
||||
SHOP_ID = "20434543575189"
|
||||
# ==============================
|
||||
LOGIN_TIMEOUT_MS = 600_000 # 10 分钟
|
||||
|
||||
|
||||
def login_and_get_cookies():
|
||||
"""用 Playwright 打开登录页,等待用户手动登录,然后提取 Cookie"""
|
||||
print("[1/3] 启动浏览器,请手动完成登录(包括验证码)...")
|
||||
def login_with_browser(login_url: str) -> dict[str, str] | None:
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
print("[错误] 未安装 playwright:pip install playwright && playwright install chromium", file=sys.stderr)
|
||||
return None
|
||||
|
||||
print("[1/3] 启动浏览器,请在窗口中完成登录(账号密码 + 验证码)...")
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=False, args=["--start-maximized"])
|
||||
context = browser.new_context(no_viewport=True)
|
||||
page = context.new_page()
|
||||
page.goto(login_url)
|
||||
|
||||
page.goto(LOGIN_URL)
|
||||
|
||||
# 等待用户登录成功(检测页面跳转到 MMS 首页)
|
||||
print("[2/3] 等待登录完成...(请在浏览器中输入账号密码并完成验证码)")
|
||||
print("[2/3] 等待登录完成(最多 10 分钟)...")
|
||||
try:
|
||||
page.wait_for_url("**/MMS**", timeout=300_000)
|
||||
print(" 登录成功!正在提取 Cookie...")
|
||||
page.wait_for_url("**/MMS**", timeout=LOGIN_TIMEOUT_MS)
|
||||
print(" 登录成功,正在提取 Cookie...")
|
||||
except Exception:
|
||||
print(" 等待超时(5分钟),请重新运行脚本。")
|
||||
print(" 等待超时,已退出。请重新运行。", file=sys.stderr)
|
||||
browser.close()
|
||||
return None
|
||||
|
||||
# 确保访问过目标页面(触发必要的 Cookie 设置)
|
||||
page.goto(MMS_BASE)
|
||||
page.wait_for_load_state("networkidle")
|
||||
# 访问一次首页,确保必要的 Cookie 都下发了
|
||||
try:
|
||||
page.goto(MMS_BASE)
|
||||
page.wait_for_load_state("networkidle", timeout=30_000)
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
|
||||
# 提取所有 Cookie
|
||||
playwright_cookies = context.cookies()
|
||||
cookies = {c["name"]: c["value"] for c in playwright_cookies}
|
||||
|
||||
print(f" 获取到 {len(cookies)} 个 Cookie")
|
||||
cookies = {c["name"]: c["value"] for c in context.cookies()}
|
||||
browser.close()
|
||||
return cookies
|
||||
|
||||
print(f"[3/3] 提取到 {len(cookies)} 个 Cookie")
|
||||
return cookies
|
||||
|
||||
|
||||
def fetch_order_list(cookies, page=1, page_size=20):
|
||||
"""调用快捷买单订单列表接口"""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Referer": "https://fs.szfx.com/MMS",
|
||||
"Origin": "https://fs.szfx.com",
|
||||
}
|
||||
|
||||
payload = {
|
||||
"shopId": SHOP_ID,
|
||||
"page": page,
|
||||
"pageSize": page_size,
|
||||
}
|
||||
|
||||
print(f"[3/3] 正在请求订单数据(第{page}页)...")
|
||||
resp = requests.post(DATA_API, json=payload, headers=headers, cookies=cookies, timeout=30)
|
||||
def verify(cookies: dict[str, str]) -> bool:
|
||||
"""本地先验一次,避免把坏 Cookie 推上去。"""
|
||||
import requests as _rq
|
||||
resp = _rq.post(
|
||||
"https://fs.szfx.com/saasmerchant/pcweb/order/quickpayorder/list",
|
||||
json={"shopId": "20434543575189", "page": 1, "pageSize": 1},
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Referer": MMS_BASE,
|
||||
"Origin": "https://fs.szfx.com",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
|
||||
},
|
||||
cookies=cookies,
|
||||
timeout=20,
|
||||
)
|
||||
data = resp.json()
|
||||
|
||||
if data.get("errno") == 0:
|
||||
print(" 请求成功!")
|
||||
return data.get("data")
|
||||
else:
|
||||
print(f" 请求失败: errno={data.get('errno')}, errmsg={data.get('errmsg')}")
|
||||
return None
|
||||
orders = (data.get("data") or {}).get("list") or []
|
||||
latest = orders[0].get("orderId") if orders else None
|
||||
print(f" Cookie 校验通过(最新订单 {latest})")
|
||||
return True
|
||||
print(f" Cookie 校验失败:errno={data.get('errno')} {data.get('errmsg')}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def save_data(data, filename="order_data.json"):
|
||||
"""保存数据到文件"""
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" 数据已保存到 {filename}")
|
||||
def upload(server: str, token: str, cookies: dict[str, str]) -> bool:
|
||||
url = server.rstrip("/") + "/api/cookies"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Auth-Token"] = token
|
||||
print(f" 上传到 {url} ...")
|
||||
try:
|
||||
resp = requests.post(url, json={"cookies": cookies}, headers=headers, timeout=20)
|
||||
except requests.RequestException as exc:
|
||||
print(f" 上传失败(网络):{exc}", file=sys.stderr)
|
||||
return False
|
||||
print(f" HTTP {resp.status_code} {resp.text[:300]}")
|
||||
return resp.status_code == 200
|
||||
|
||||
|
||||
def main():
|
||||
# 第一步:登录获取 Cookie
|
||||
cookies = login_and_get_cookies()
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="丰享商家端登录并同步 Cookie")
|
||||
parser.add_argument("--upload", action="store_true", help="登录成功后上传到服务器")
|
||||
parser.add_argument("--server", default=None, help="服务器地址,如 http://192.168.8.210:52100")
|
||||
parser.add_argument("--token", default=None, help="管理接口 token")
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg = load_config()
|
||||
cookies = login_with_browser(cfg.api.login_url)
|
||||
if not cookies:
|
||||
return
|
||||
return 1
|
||||
|
||||
# 保存 Cookie 以便下次复用
|
||||
with open("cookies.json", "w", encoding="utf-8") as f:
|
||||
if not verify(cookies):
|
||||
print("Cookie 不可用,未保存。", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
out = Path(cfg.paths.cookies)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump(cookies, f, ensure_ascii=False, indent=2)
|
||||
print(" Cookie 已保存到 cookies.json(下次可直接使用)\n")
|
||||
print(f" 已保存到 {out}")
|
||||
|
||||
# 第二步:获取订单数据
|
||||
result = fetch_order_list(cookies)
|
||||
if result:
|
||||
save_data(result)
|
||||
# 打印摘要
|
||||
if isinstance(result, dict):
|
||||
print(f"\n 数据摘要:")
|
||||
for k, v in result.items():
|
||||
if not isinstance(v, (list, dict)):
|
||||
print(f" {k}: {v}")
|
||||
elif isinstance(v, list):
|
||||
print(f" {k}: 共 {len(v)} 条记录")
|
||||
if args.upload:
|
||||
import os
|
||||
server = args.server or os.environ.get("FX_SERVER")
|
||||
token = args.token or os.environ.get("FX_ADMIN_TOKEN") or cfg.admin.token
|
||||
if not server:
|
||||
print("[错误] 未指定服务器:用 --server 或环境变量 FX_SERVER", file=sys.stderr)
|
||||
return 3
|
||||
return 0 if upload(server, token, cookies) else 4
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user