mirror of
https://gitee.com/houhuan/TrendRadar.git
synced 2026-05-01 01:12:42 +08:00
v4.0.0 大大大更新
This commit is contained in:
sansan
@@ -25,7 +25,6 @@ def calculate_news_weight(news_data: Dict, rank_threshold: int = 5) -> float:
|
||||
"""
|
||||
计算新闻权重（用于排序）
|
||||
|
||||
基于 main.py 的权重算法实现，综合考虑：
|
||||
- 排名权重 (60%)：新闻在榜单中的排名
|
||||
- 频次权重 (30%)：新闻出现的次数
|
||||
- 热度权重 (10%)：高排名出现的比例
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
存储同步工具
|
||||
|
||||
实现从远程存储拉取数据到本地、获取存储状态、列出可用日期等功能。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from ..utils.errors import MCPError
|
||||
|
||||
|
||||
class StorageSyncTools:
|
||||
"""存储同步工具类"""
|
||||
|
||||
def __init__(self, project_root: str = None):
|
||||
"""
|
||||
初始化存储同步工具
|
||||
|
||||
Args:
|
||||
project_root: 项目根目录
|
||||
"""
|
||||
if project_root:
|
||||
self.project_root = Path(project_root)
|
||||
else:
|
||||
current_file = Path(__file__)
|
||||
self.project_root = current_file.parent.parent.parent
|
||||
|
||||
self._config = None
|
||||
self._remote_backend = None
|
||||
|
||||
def _load_config(self) -> dict:
|
||||
"""加载配置文件"""
|
||||
if self._config is None:
|
||||
config_path = self.project_root / "config" / "config.yaml"
|
||||
if config_path.exists():
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
self._config = yaml.safe_load(f)
|
||||
else:
|
||||
self._config = {}
|
||||
return self._config
|
||||
|
||||
def _get_storage_config(self) -> dict:
|
||||
"""获取存储配置"""
|
||||
config = self._load_config()
|
||||
return config.get("storage", {})
|
||||
|
||||
def _get_remote_config(self) -> dict:
|
||||
"""
|
||||
获取远程存储配置（合并配置文件和环境变量）
|
||||
"""
|
||||
storage_config = self._get_storage_config()
|
||||
remote_config = storage_config.get("remote", {})
|
||||
|
||||
return {
|
||||
"endpoint_url": remote_config.get("endpoint_url") or os.environ.get("S3_ENDPOINT_URL", ""),
|
||||
"bucket_name": remote_config.get("bucket_name") or os.environ.get("S3_BUCKET_NAME", ""),
|
||||
"access_key_id": remote_config.get("access_key_id") or os.environ.get("S3_ACCESS_KEY_ID", ""),
|
||||
"secret_access_key": remote_config.get("secret_access_key") or os.environ.get("S3_SECRET_ACCESS_KEY", ""),
|
||||
"region": remote_config.get("region") or os.environ.get("S3_REGION", ""),
|
||||
}
|
||||
|
||||
def _has_remote_config(self) -> bool:
|
||||
"""检查是否有有效的远程存储配置"""
|
||||
config = self._get_remote_config()
|
||||
return bool(
|
||||
config.get("bucket_name") and
|
||||
config.get("access_key_id") and
|
||||
config.get("secret_access_key") and
|
||||
config.get("endpoint_url")
|
||||
)
|
||||
|
||||
def _get_remote_backend(self):
|
||||
"""获取远程存储后端实例"""
|
||||
if self._remote_backend is not None:
|
||||
return self._remote_backend
|
||||
|
||||
if not self._has_remote_config():
|
||||
return None
|
||||
|
||||
try:
|
||||
from trendradar.storage.remote import RemoteStorageBackend
|
||||
|
||||
remote_config = self._get_remote_config()
|
||||
config = self._load_config()
|
||||
timezone = config.get("app", {}).get("timezone", "Asia/Shanghai")
|
||||
|
||||
self._remote_backend = RemoteStorageBackend(
|
||||
bucket_name=remote_config["bucket_name"],
|
||||
access_key_id=remote_config["access_key_id"],
|
||||
secret_access_key=remote_config["secret_access_key"],
|
||||
endpoint_url=remote_config["endpoint_url"],
|
||||
region=remote_config.get("region", ""),
|
||||
timezone=timezone,
|
||||
)
|
||||
return self._remote_backend
|
||||
except ImportError:
|
||||
print("[存储同步] 远程存储后端需要安装 boto3: pip install boto3")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"[存储同步] 创建远程后端失败: {e}")
|
||||
return None
|
||||
|
||||
def _get_local_data_dir(self) -> Path:
|
||||
"""获取本地数据目录"""
|
||||
storage_config = self._get_storage_config()
|
||||
local_config = storage_config.get("local", {})
|
||||
data_dir = local_config.get("data_dir", "output")
|
||||
return self.project_root / data_dir
|
||||
|
||||
def _parse_date_folder_name(self, folder_name: str) -> Optional[datetime]:
|
||||
"""
|
||||
解析日期文件夹名称（兼容中文和 ISO 格式）
|
||||
|
||||
支持两种格式：
|
||||
- 中文格式：YYYY年MM月DD日
|
||||
- ISO 格式：YYYY-MM-DD
|
||||
"""
|
||||
# 尝试 ISO 格式
|
||||
iso_match = re.match(r'(\d{4})-(\d{2})-(\d{2})', folder_name)
|
||||
if iso_match:
|
||||
try:
|
||||
return datetime(
|
||||
int(iso_match.group(1)),
|
||||
int(iso_match.group(2)),
|
||||
int(iso_match.group(3))
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 尝试中文格式
|
||||
chinese_match = re.match(r'(\d{4})年(\d{2})月(\d{2})日', folder_name)
|
||||
if chinese_match:
|
||||
try:
|
||||
return datetime(
|
||||
int(chinese_match.group(1)),
|
||||
int(chinese_match.group(2)),
|
||||
int(chinese_match.group(3))
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _get_local_dates(self) -> List[str]:
|
||||
"""获取本地可用的日期列表"""
|
||||
local_dir = self._get_local_data_dir()
|
||||
dates = []
|
||||
|
||||
if not local_dir.exists():
|
||||
return dates
|
||||
|
||||
for item in local_dir.iterdir():
|
||||
if item.is_dir() and not item.name.startswith('.'):
|
||||
folder_date = self._parse_date_folder_name(item.name)
|
||||
if folder_date:
|
||||
dates.append(folder_date.strftime("%Y-%m-%d"))
|
||||
|
||||
return sorted(dates, reverse=True)
|
||||
|
||||
def _calculate_dir_size(self, path: Path) -> int:
|
||||
"""计算目录大小（字节）"""
|
||||
total_size = 0
|
||||
if path.exists():
|
||||
for item in path.rglob("*"):
|
||||
if item.is_file():
|
||||
total_size += item.stat().st_size
|
||||
return total_size
|
||||
|
||||
def sync_from_remote(self, days: int = 7) -> Dict:
|
||||
"""
|
||||
从远程存储拉取数据到本地
|
||||
|
||||
Args:
|
||||
days: 拉取最近 N 天的数据，默认 7 天
|
||||
|
||||
Returns:
|
||||
同步结果字典
|
||||
"""
|
||||
try:
|
||||
# 检查远程配置
|
||||
if not self._has_remote_config():
|
||||
return {
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "REMOTE_NOT_CONFIGURED",
|
||||
"message": "未配置远程存储",
|
||||
"suggestion": "请在 config/config.yaml 中配置 storage.remote 或设置环境变量"
|
||||
}
|
||||
}
|
||||
|
||||
# 获取远程后端
|
||||
remote_backend = self._get_remote_backend()
|
||||
if remote_backend is None:
|
||||
return {
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "REMOTE_BACKEND_FAILED",
|
||||
"message": "无法创建远程存储后端",
|
||||
"suggestion": "请检查远程存储配置和 boto3 是否已安装"
|
||||
}
|
||||
}
|
||||
|
||||
# 获取本地数据目录
|
||||
local_dir = self._get_local_data_dir()
|
||||
local_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 获取远程可用日期
|
||||
remote_dates = remote_backend.list_remote_dates()
|
||||
|
||||
# 获取本地已有日期
|
||||
local_dates = set(self._get_local_dates())
|
||||
|
||||
# 计算需要拉取的日期（最近 N 天）
|
||||
from trendradar.utils.time import get_configured_time
|
||||
config = self._load_config()
|
||||
timezone = config.get("app", {}).get("timezone", "Asia/Shanghai")
|
||||
now = get_configured_time(timezone)
|
||||
|
||||
target_dates = []
|
||||
for i in range(days):
|
||||
date = now - timedelta(days=i)
|
||||
date_str = date.strftime("%Y-%m-%d")
|
||||
if date_str in remote_dates:
|
||||
target_dates.append(date_str)
|
||||
|
||||
# 执行拉取
|
||||
synced_dates = []
|
||||
skipped_dates = []
|
||||
failed_dates = []
|
||||
|
||||
for date_str in target_dates:
|
||||
# 检查本地是否已存在
|
||||
if date_str in local_dates:
|
||||
skipped_dates.append(date_str)
|
||||
continue
|
||||
|
||||
# 拉取单个日期
|
||||
try:
|
||||
local_date_dir = local_dir / date_str
|
||||
local_db_path = local_date_dir / "news.db"
|
||||
remote_key = f"news/{date_str}.db"
|
||||
|
||||
local_date_dir.mkdir(parents=True, exist_ok=True)
|
||||
remote_backend.s3_client.download_file(
|
||||
remote_backend.bucket_name,
|
||||
remote_key,
|
||||
str(local_db_path)
|
||||
)
|
||||
synced_dates.append(date_str)
|
||||
print(f"[存储同步] 已拉取: {date_str}")
|
||||
except Exception as e:
|
||||
failed_dates.append({"date": date_str, "error": str(e)})
|
||||
print(f"[存储同步] 拉取失败 ({date_str}): {e}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"synced_files": len(synced_dates),
|
||||
"synced_dates": synced_dates,
|
||||
"skipped_dates": skipped_dates,
|
||||
"failed_dates": failed_dates,
|
||||
"message": f"成功同步 {len(synced_dates)} 天数据" + (
|
||||
f"，跳过 {len(skipped_dates)} 天（本地已存在）" if skipped_dates else ""
|
||||
) + (
|
||||
f"，失败 {len(failed_dates)} 天" if failed_dates else ""
|
||||
)
|
||||
}
|
||||
|
||||
except MCPError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": e.to_dict()
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": str(e)
|
||||
}
|
||||
}
|
||||
|
||||
def get_storage_status(self) -> Dict:
|
||||
"""
|
||||
获取存储配置和状态
|
||||
|
||||
Returns:
|
||||
存储状态字典
|
||||
"""
|
||||
try:
|
||||
storage_config = self._get_storage_config()
|
||||
config = self._load_config()
|
||||
|
||||
# 本地存储状态
|
||||
local_config = storage_config.get("local", {})
|
||||
local_dir = self._get_local_data_dir()
|
||||
local_size = self._calculate_dir_size(local_dir)
|
||||
local_dates = self._get_local_dates()
|
||||
|
||||
local_status = {
|
||||
"data_dir": local_config.get("data_dir", "output"),
|
||||
"retention_days": local_config.get("retention_days", 0),
|
||||
"total_size": f"{local_size / 1024 / 1024:.2f} MB",
|
||||
"total_size_bytes": local_size,
|
||||
"date_count": len(local_dates),
|
||||
"earliest_date": local_dates[-1] if local_dates else None,
|
||||
"latest_date": local_dates[0] if local_dates else None,
|
||||
}
|
||||
|
||||
# 远程存储状态
|
||||
remote_config = storage_config.get("remote", {})
|
||||
has_remote = self._has_remote_config()
|
||||
|
||||
remote_status = {
|
||||
"configured": has_remote,
|
||||
"retention_days": remote_config.get("retention_days", 0),
|
||||
}
|
||||
|
||||
if has_remote:
|
||||
merged_config = self._get_remote_config()
|
||||
# 脱敏显示
|
||||
endpoint = merged_config.get("endpoint_url", "")
|
||||
bucket = merged_config.get("bucket_name", "")
|
||||
remote_status["endpoint_url"] = endpoint
|
||||
remote_status["bucket_name"] = bucket
|
||||
|
||||
# 尝试获取远程日期列表
|
||||
remote_backend = self._get_remote_backend()
|
||||
if remote_backend:
|
||||
try:
|
||||
remote_dates = remote_backend.list_remote_dates()
|
||||
remote_status["date_count"] = len(remote_dates)
|
||||
remote_status["earliest_date"] = remote_dates[-1] if remote_dates else None
|
||||
remote_status["latest_date"] = remote_dates[0] if remote_dates else None
|
||||
except Exception as e:
|
||||
remote_status["error"] = str(e)
|
||||
|
||||
# 拉取配置状态
|
||||
pull_config = storage_config.get("pull", {})
|
||||
pull_status = {
|
||||
"enabled": pull_config.get("enabled", False),
|
||||
"days": pull_config.get("days", 7),
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"backend": storage_config.get("backend", "auto"),
|
||||
"local": local_status,
|
||||
"remote": remote_status,
|
||||
"pull": pull_status,
|
||||
}
|
||||
|
||||
except MCPError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": e.to_dict()
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": str(e)
|
||||
}
|
||||
}
|
||||
|
||||
def list_available_dates(self, source: str = "both") -> Dict:
|
||||
"""
|
||||
列出可用的日期范围
|
||||
|
||||
Args:
|
||||
source: 数据来源
|
||||
- "local": 仅本地
|
||||
- "remote": 仅远程
|
||||
- "both": 两者都列出（默认）
|
||||
|
||||
Returns:
|
||||
日期列表字典
|
||||
"""
|
||||
try:
|
||||
result = {
|
||||
"success": True,
|
||||
}
|
||||
|
||||
# 本地日期
|
||||
if source in ("local", "both"):
|
||||
local_dates = self._get_local_dates()
|
||||
result["local"] = {
|
||||
"dates": local_dates,
|
||||
"count": len(local_dates),
|
||||
"earliest": local_dates[-1] if local_dates else None,
|
||||
"latest": local_dates[0] if local_dates else None,
|
||||
}
|
||||
|
||||
# 远程日期
|
||||
if source in ("remote", "both"):
|
||||
if not self._has_remote_config():
|
||||
result["remote"] = {
|
||||
"configured": False,
|
||||
"dates": [],
|
||||
"count": 0,
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
"error": "未配置远程存储"
|
||||
}
|
||||
else:
|
||||
remote_backend = self._get_remote_backend()
|
||||
if remote_backend:
|
||||
try:
|
||||
remote_dates = remote_backend.list_remote_dates()
|
||||
result["remote"] = {
|
||||
"configured": True,
|
||||
"dates": remote_dates,
|
||||
"count": len(remote_dates),
|
||||
"earliest": remote_dates[-1] if remote_dates else None,
|
||||
"latest": remote_dates[0] if remote_dates else None,
|
||||
}
|
||||
except Exception as e:
|
||||
result["remote"] = {
|
||||
"configured": True,
|
||||
"dates": [],
|
||||
"count": 0,
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
"error": str(e)
|
||||
}
|
||||
else:
|
||||
result["remote"] = {
|
||||
"configured": True,
|
||||
"dates": [],
|
||||
"count": 0,
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
"error": "无法创建远程存储后端"
|
||||
}
|
||||
|
||||
# 如果同时查询两者，计算差异
|
||||
if source == "both" and "local" in result and "remote" in result:
|
||||
local_set = set(result["local"]["dates"])
|
||||
remote_set = set(result["remote"].get("dates", []))
|
||||
|
||||
result["comparison"] = {
|
||||
"only_local": sorted(list(local_set - remote_set), reverse=True),
|
||||
"only_remote": sorted(list(remote_set - local_set), reverse=True),
|
||||
"both": sorted(list(local_set & remote_set), reverse=True),
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
except MCPError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": e.to_dict()
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "INTERNAL_ERROR",
|
||||
"message": str(e)
|
||||
}
|
||||
}
|
||||
+93
-190
@@ -87,13 +87,13 @@ class SystemManagementTools:
|
||||
>>> print(result['saved_files'])
|
||||
"""
|
||||
try:
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import requests
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
import yaml
|
||||
from trendradar.crawler.fetcher import DataFetcher
|
||||
from trendradar.storage.local import LocalStorageBackend
|
||||
from trendradar.storage.base import convert_crawl_results_to_news_data
|
||||
from trendradar.utils.time import get_configured_time, format_date_folder, format_time_filename
|
||||
from ..services.cache_service import get_cache
|
||||
|
||||
# 参数验证
|
||||
platforms = validate_platforms(platforms)
|
||||
@@ -129,9 +129,6 @@ class SystemManagementTools:
|
||||
else:
|
||||
target_platforms = all_platforms
|
||||
|
||||
# 获取请求间隔
|
||||
request_interval = config_data.get("crawler", {}).get("request_interval", 100)
|
||||
|
||||
# 构建平台ID列表
|
||||
ids = []
|
||||
for platform in target_platforms:
|
||||
@@ -142,87 +139,82 @@ class SystemManagementTools:
|
||||
|
||||
print(f"开始临时爬取，平台: {[p.get('name', p['id']) for p in target_platforms]}")
|
||||
|
||||
# 爬取数据
|
||||
results = {}
|
||||
id_to_name = {}
|
||||
failed_ids = []
|
||||
# 初始化数据获取器
|
||||
crawler_config = config_data.get("crawler", {})
|
||||
proxy_url = None
|
||||
if crawler_config.get("use_proxy"):
|
||||
proxy_url = crawler_config.get("proxy_url")
|
||||
|
||||
fetcher = DataFetcher(proxy_url=proxy_url)
|
||||
request_interval = crawler_config.get("request_interval", 100)
|
||||
|
||||
for i, id_info in enumerate(ids):
|
||||
if isinstance(id_info, tuple):
|
||||
id_value, name = id_info
|
||||
else:
|
||||
id_value = id_info
|
||||
name = id_value
|
||||
# 执行爬取
|
||||
results, id_to_name, failed_ids = fetcher.crawl_websites(
|
||||
ids_list=ids,
|
||||
request_interval=request_interval
|
||||
)
|
||||
|
||||
id_to_name[id_value] = name
|
||||
# 获取当前时间（统一使用 trendradar 的时间工具）
|
||||
# 从配置中读取时区，默认为 Asia/Shanghai
|
||||
timezone = config_data.get("app", {}).get("timezone", "Asia/Shanghai")
|
||||
current_time = get_configured_time(timezone)
|
||||
crawl_date = format_date_folder(None, timezone)
|
||||
crawl_time_str = format_time_filename(timezone)
|
||||
|
||||
# 构建请求URL
|
||||
url = f"https://newsnow.busiyi.world/api/s?id={id_value}&latest"
|
||||
# 转换为标准数据模型
|
||||
news_data = convert_crawl_results_to_news_data(
|
||||
results=results,
|
||||
id_to_name=id_to_name,
|
||||
failed_ids=failed_ids,
|
||||
crawl_time=crawl_time_str,
|
||||
crawl_date=crawl_date
|
||||
)
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Connection": "keep-alive",
|
||||
"Cache-Control": "no-cache",
|
||||
}
|
||||
# 初始化存储后端
|
||||
storage = LocalStorageBackend(
|
||||
data_dir=str(self.project_root / "output"),
|
||||
enable_txt=True,
|
||||
enable_html=True,
|
||||
timezone=timezone
|
||||
)
|
||||
|
||||
# 重试机制
|
||||
max_retries = 2
|
||||
retries = 0
|
||||
success = False
|
||||
# 尝试持久化数据
|
||||
save_success = False
|
||||
save_error_msg = ""
|
||||
saved_files = {}
|
||||
|
||||
while retries <= max_retries and not success:
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
try:
|
||||
# 1. 保存到 SQLite (核心持久化)
|
||||
if storage.save_news_data(news_data):
|
||||
save_success = True
|
||||
|
||||
# 2. 如果请求保存到本地，生成 TXT/HTML 快照
|
||||
if save_to_local:
|
||||
# 保存 TXT
|
||||
txt_path = storage.save_txt_snapshot(news_data)
|
||||
if txt_path:
|
||||
saved_files["txt"] = txt_path
|
||||
|
||||
data_text = response.text
|
||||
data_json = json.loads(data_text)
|
||||
# 保存 HTML (使用简化版生成器)
|
||||
html_content = self._generate_simple_html(results, id_to_name, failed_ids, current_time)
|
||||
html_filename = f"{crawl_time_str}.html"
|
||||
html_path = storage.save_html_report(html_content, html_filename)
|
||||
if html_path:
|
||||
saved_files["html"] = html_path
|
||||
|
||||
status = data_json.get("status", "未知")
|
||||
if status not in ["success", "cache"]:
|
||||
raise ValueError(f"响应状态异常: {status}")
|
||||
except Exception as e:
|
||||
# 捕获所有保存错误（特别是 Docker 只读卷导致的 PermissionError）
|
||||
print(f"[System] 数据保存失败: {e}")
|
||||
save_success = False
|
||||
save_error_msg = str(e)
|
||||
|
||||
status_info = "最新数据" if status == "success" else "缓存数据"
|
||||
print(f"获取 {id_value} 成功（{status_info}）")
|
||||
# 3. 清除缓存，确保下次查询获取最新数据
|
||||
# 即使保存失败，内存中的数据可能已经通过其他方式更新，或者是临时的
|
||||
get_cache().clear()
|
||||
print("[System] 缓存已清除")
|
||||
|
||||
# 解析数据
|
||||
results[id_value] = {}
|
||||
for index, item in enumerate(data_json.get("items", []), 1):
|
||||
title = item["title"]
|
||||
url_link = item.get("url", "")
|
||||
mobile_url = item.get("mobileUrl", "")
|
||||
|
||||
if title in results[id_value]:
|
||||
results[id_value][title]["ranks"].append(index)
|
||||
else:
|
||||
results[id_value][title] = {
|
||||
"ranks": [index],
|
||||
"url": url_link,
|
||||
"mobileUrl": mobile_url,
|
||||
}
|
||||
|
||||
success = True
|
||||
|
||||
except Exception as e:
|
||||
retries += 1
|
||||
if retries <= max_retries:
|
||||
wait_time = random.uniform(3, 5)
|
||||
print(f"请求 {id_value} 失败: {e}. {wait_time:.2f}秒后重试...")
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
print(f"请求 {id_value} 失败: {e}")
|
||||
failed_ids.append(id_value)
|
||||
|
||||
# 请求间隔
|
||||
if i < len(ids) - 1:
|
||||
actual_interval = request_interval + random.randint(-10, 20)
|
||||
actual_interval = max(50, actual_interval)
|
||||
time.sleep(actual_interval / 1000)
|
||||
|
||||
# 格式化返回数据
|
||||
news_data = []
|
||||
# 构建返回结果
|
||||
news_response_data = []
|
||||
for platform_id, titles_data in results.items():
|
||||
platform_name = id_to_name.get(platform_id, platform_id)
|
||||
for title, info in titles_data.items():
|
||||
@@ -230,131 +222,42 @@ class SystemManagementTools:
|
||||
"platform_id": platform_id,
|
||||
"platform_name": platform_name,
|
||||
"title": title,
|
||||
"ranks": info["ranks"]
|
||||
"ranks": info.get("ranks", [])
|
||||
}
|
||||
|
||||
# 条件性添加 URL 字段
|
||||
if include_url:
|
||||
news_item["url"] = info.get("url", "")
|
||||
news_item["mobile_url"] = info.get("mobileUrl", "")
|
||||
news_response_data.append(news_item)
|
||||
|
||||
news_data.append(news_item)
|
||||
|
||||
# 获取北京时间
|
||||
beijing_tz = pytz.timezone("Asia/Shanghai")
|
||||
now = datetime.now(beijing_tz)
|
||||
|
||||
# 构建返回结果
|
||||
result = {
|
||||
"success": True,
|
||||
"task_id": f"crawl_{int(time.time())}",
|
||||
"status": "completed",
|
||||
"crawl_time": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"crawl_time": current_time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"platforms": list(results.keys()),
|
||||
"total_news": len(news_data),
|
||||
"total_news": len(news_response_data),
|
||||
"failed_platforms": failed_ids,
|
||||
"data": news_data,
|
||||
"saved_to_local": save_to_local
|
||||
"data": news_response_data,
|
||||
"saved_to_local": save_success and save_to_local
|
||||
}
|
||||
|
||||
# 如果需要持久化，调用保存逻辑
|
||||
if save_to_local:
|
||||
try:
|
||||
import re
|
||||
|
||||
# 辅助函数：清理标题
|
||||
def clean_title(title: str) -> str:
|
||||
"""清理标题中的特殊字符"""
|
||||
if not isinstance(title, str):
|
||||
title = str(title)
|
||||
cleaned_title = title.replace("\n", " ").replace("\r", " ")
|
||||
cleaned_title = re.sub(r"\s+", " ", cleaned_title)
|
||||
cleaned_title = cleaned_title.strip()
|
||||
return cleaned_title
|
||||
|
||||
# 辅助函数：创建目录
|
||||
def ensure_directory_exists(directory: str):
|
||||
"""确保目录存在"""
|
||||
Path(directory).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 格式化日期和时间
|
||||
date_folder = now.strftime("%Y年%m月%d日")
|
||||
time_filename = now.strftime("%H时%M分")
|
||||
|
||||
# 创建 txt 文件路径
|
||||
txt_dir = self.project_root / "output" / date_folder / "txt"
|
||||
ensure_directory_exists(str(txt_dir))
|
||||
txt_file_path = txt_dir / f"{time_filename}.txt"
|
||||
|
||||
# 创建 html 文件路径
|
||||
html_dir = self.project_root / "output" / date_folder / "html"
|
||||
ensure_directory_exists(str(html_dir))
|
||||
html_file_path = html_dir / f"{time_filename}.html"
|
||||
|
||||
# 保存 txt 文件（按照 main.py 的格式）
|
||||
with open(txt_file_path, "w", encoding="utf-8") as f:
|
||||
for id_value, title_data in results.items():
|
||||
# id | name 或 id
|
||||
name = id_to_name.get(id_value)
|
||||
if name and name != id_value:
|
||||
f.write(f"{id_value} | {name}\n")
|
||||
else:
|
||||
f.write(f"{id_value}\n")
|
||||
|
||||
# 按排名排序标题
|
||||
sorted_titles = []
|
||||
for title, info in title_data.items():
|
||||
cleaned = clean_title(title)
|
||||
if isinstance(info, dict):
|
||||
ranks = info.get("ranks", [])
|
||||
url = info.get("url", "")
|
||||
mobile_url = info.get("mobileUrl", "")
|
||||
else:
|
||||
ranks = info if isinstance(info, list) else []
|
||||
url = ""
|
||||
mobile_url = ""
|
||||
|
||||
rank = ranks[0] if ranks else 1
|
||||
sorted_titles.append((rank, cleaned, url, mobile_url))
|
||||
|
||||
sorted_titles.sort(key=lambda x: x[0])
|
||||
|
||||
for rank, cleaned, url, mobile_url in sorted_titles:
|
||||
line = f"{rank}. {cleaned}"
|
||||
if url:
|
||||
line += f" [URL:{url}]"
|
||||
if mobile_url:
|
||||
line += f" [MOBILE:{mobile_url}]"
|
||||
f.write(line + "\n")
|
||||
|
||||
f.write("\n")
|
||||
|
||||
if failed_ids:
|
||||
f.write("==== 以下ID请求失败 ====\n")
|
||||
for id_value in failed_ids:
|
||||
f.write(f"{id_value}\n")
|
||||
|
||||
# 保存 html 文件（简化版）
|
||||
html_content = self._generate_simple_html(results, id_to_name, failed_ids, now)
|
||||
with open(html_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
|
||||
print(f"数据已保存到:")
|
||||
print(f" TXT: {txt_file_path}")
|
||||
print(f" HTML: {html_file_path}")
|
||||
|
||||
result["saved_files"] = {
|
||||
"txt": str(txt_file_path),
|
||||
"html": str(html_file_path)
|
||||
}
|
||||
result["note"] = "数据已持久化到 output 文件夹"
|
||||
|
||||
except Exception as e:
|
||||
print(f"保存文件失败: {e}")
|
||||
result["save_error"] = str(e)
|
||||
result["note"] = "爬取成功但保存失败，数据仅在内存中"
|
||||
if save_success:
|
||||
if save_to_local:
|
||||
result["saved_files"] = saved_files
|
||||
result["note"] = "数据已保存到 SQLite 数据库及 output 文件夹"
|
||||
else:
|
||||
result["note"] = "数据已保存到 SQLite 数据库 (仅内存中返回结果，未生成TXT快照)"
|
||||
else:
|
||||
result["note"] = "临时爬取结果，未持久化到output文件夹"
|
||||
# 明确告知用户保存失败
|
||||
result["saved_to_local"] = False
|
||||
result["save_error"] = save_error_msg
|
||||
if "Read-only file system" in save_error_msg or "Permission denied" in save_error_msg:
|
||||
result["note"] = "爬取成功，但无法写入数据库（Docker只读模式）。数据仅在本次返回中有效。"
|
||||
else:
|
||||
result["note"] = f"爬取成功但保存失败: {save_error_msg}"
|
||||
|
||||
# 清理资源
|
||||
storage.cleanup()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
Reference in New Issue
Block a user