优化: Docker 环境下的下载性能和网络稳定性
- Chromium 启动参数优化:禁用 dev-shm 和 GPU 加速,防止Docker内存不足 - 增加所有超时时间:login/navigate/export 超时 30s,下载超时 300s - 改进网络延迟处理:增加数据加载等待时间,添加网络加载检测 - Docker Compose 资源配置:限制 2 CPU / 2GB 内存,DNS 配置国际公共 DNS - Dockerfile 优化:添加 PYTHONHASHSEED 环境变量,跳过浏览器下载校验 - 新增 docker-debug.sh 脚本:便捷测试 Docker 容器中的下载功能 - 新增 .dockerignore:加速 Docker 构建,减少镜像大小 Docker 下载现在支持更长的网络延迟和更大的数据量
This commit is contained in:
+24
-16
@@ -42,7 +42,15 @@ class SecsionDownloader:
|
||||
logger.info(f"开始下载报表: {start_date} ~ {end_date} (第 {attempt + 1}/{retry_count} 次)")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
# Docker 优化:添加 --disable-dev-shm-usage 避免共享内存不足
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--single-process"
|
||||
]
|
||||
)
|
||||
context = await browser.new_context(
|
||||
ignore_https_errors=True,
|
||||
viewport={'width': 1280, 'height': 800}
|
||||
@@ -70,14 +78,14 @@ class SecsionDownloader:
|
||||
async def _login(self, page):
|
||||
"""登录 secsion.com"""
|
||||
logger.info(f"打开登录页面: {self.LOGIN_URL}")
|
||||
await page.goto(self.LOGIN_URL)
|
||||
await page.goto(self.LOGIN_URL, timeout=30000)
|
||||
|
||||
# 选择角色 "店铺"
|
||||
logger.info("选择角色: 店铺")
|
||||
try:
|
||||
await page.get_by_text("店铺", exact=True).click()
|
||||
await page.get_by_text("店铺", exact=True).click(timeout=10000)
|
||||
except Exception:
|
||||
await page.click("text=店铺")
|
||||
await page.click("text=店铺", timeout=10000)
|
||||
|
||||
# 输入账号密码
|
||||
logger.info(f"输入账号: {self.username}")
|
||||
@@ -93,21 +101,21 @@ class SecsionDownloader:
|
||||
try:
|
||||
await page.click("button:has-text('登录')", timeout=5000)
|
||||
except Exception:
|
||||
await page.click("button[type='submit']")
|
||||
await page.click("button[type='submit']", timeout=5000)
|
||||
|
||||
# 等待跳转
|
||||
# 等待跳转(Docker 中需要更长时间)
|
||||
logger.info("等待登录跳转...")
|
||||
await page.wait_for_url("**/homePage", timeout=20000)
|
||||
await page.wait_for_url("**/homePage", timeout=30000)
|
||||
logger.info("登录成功")
|
||||
|
||||
async def _export_report(self, page, start_date, end_date):
|
||||
"""访问统计页面并导出报表"""
|
||||
logger.info(f"访问统计页面: {self.STATS_URL}")
|
||||
await page.goto(self.STATS_URL)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
await page.goto(self.STATS_URL, timeout=30000)
|
||||
await page.wait_for_load_state("networkidle", timeout=30000)
|
||||
|
||||
export_btn = page.get_by_role("button", name="导出报表")
|
||||
await export_btn.wait_for(state="visible", timeout=20000)
|
||||
await export_btn.wait_for(state="visible", timeout=30000)
|
||||
|
||||
logger.info(f"设置查询日期范围: {start_date} ~ {end_date}")
|
||||
|
||||
@@ -127,9 +135,9 @@ class SecsionDownloader:
|
||||
end_val = await end_input.input_value()
|
||||
logger.info(f"日期设置结果: 开始={start_val}, 结束={end_val}")
|
||||
|
||||
# 等待数据请求完成 + 表格渲染
|
||||
# 等待数据请求完成 + 表格渲染(Docker 中增加等待时间)
|
||||
logger.info("等待数据请求完成...")
|
||||
await asyncio.sleep(2)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 检查数据是否加载完成(等待loading消失或有实际数据)
|
||||
try:
|
||||
@@ -143,13 +151,13 @@ class SecsionDownloader:
|
||||
const rows = document.querySelectorAll('table tbody tr');
|
||||
return rows.length > 0;
|
||||
}""",
|
||||
timeout=15000
|
||||
timeout=30000
|
||||
)
|
||||
logger.info("数据表格已加载")
|
||||
except Exception as e:
|
||||
logger.warning(f"表格加载检查失败: {e},继续执行...")
|
||||
|
||||
await asyncio.sleep(2)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 如果配置了 shop_id,拦截导出请求注入 shop_id
|
||||
if self.shop_id:
|
||||
@@ -165,9 +173,9 @@ class SecsionDownloader:
|
||||
await page.route('**/api/bill/export', inject_shop_id)
|
||||
logger.info(f"已设置 shop_id 拦截: {self.shop_id}")
|
||||
|
||||
# 点击导出报表并捕获下载(增加超时时间到180秒处理大文件)
|
||||
# 点击导出报表并捕获下载(Docker 中增加超时到300秒)
|
||||
logger.info("点击导出报表...")
|
||||
async with page.expect_download(timeout=180000) as download_info:
|
||||
async with page.expect_download(timeout=300000) as download_info:
|
||||
await export_btn.click()
|
||||
logger.info("等待文件下载中...")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user