mirror of
https://github.com/shufflewzc/faker2.git
synced 2025-04-23 02:48:44 +08:00
update
This commit is contained in:
Faker
parent
647db3ddad
commit
4965f31f77
116
depend.py
Normal file
116
depend.py
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
class Depend:
|
||||||
|
@staticmethod
|
||||||
|
def get_ql_path():
|
||||||
|
if re.search('/ql/data/', sys.path[0]):
|
||||||
|
return '/ql/data/'
|
||||||
|
else:
|
||||||
|
return '/ql/'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_env(env, default=None):
|
||||||
|
"""
|
||||||
|
青龙环境变量读取，支持将整数，bool类型变量转化为正常的值
|
||||||
|
Args:
|
||||||
|
env: 字符串，被读取的青龙环境变量
|
||||||
|
default: 字符串，如果找不到这个环境变量，返回的默认变量
|
||||||
|
Returns:
|
||||||
|
result 被格式化的变量
|
||||||
|
"""
|
||||||
|
if env in os.environ and os.environ[env]:
|
||||||
|
if os.environ[env] in ["True", "False"]:
|
||||||
|
return False if os.environ[env] == "False" else True
|
||||||
|
elif os.environ[env].isdigit():
|
||||||
|
return int(os.environ[env])
|
||||||
|
else:
|
||||||
|
return os.environ[env]
|
||||||
|
else:
|
||||||
|
if default:
|
||||||
|
if default in ["True", "False"]:
|
||||||
|
return False if default == "False" else True
|
||||||
|
elif default.isdigit():
|
||||||
|
return int(default)
|
||||||
|
else:
|
||||||
|
return default
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def str2list(string):
|
||||||
|
if string and string != "":
|
||||||
|
if re.search(",", string):
|
||||||
|
return string.split(",")
|
||||||
|
else:
|
||||||
|
return [string]
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def not2append(addlist, appended):
|
||||||
|
for i in addlist:
|
||||||
|
if i not in appended:
|
||||||
|
appended.append(i)
|
||||||
|
return appended
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def re_filter_list(string, filter_list):
|
||||||
|
for i in filter_list:
|
||||||
|
if re.search(i, string):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def only_check(self, pyname, pyabspath,osenv=None):
|
||||||
|
only_path = self.get_ql_path() + pyname + '_by_keven1024'
|
||||||
|
result = "☺当前脚本目录为: " + str(pyabspath) + "\n"
|
||||||
|
j_data = {
|
||||||
|
"py_path": None
|
||||||
|
}
|
||||||
|
if osenv and self.get_env(osenv):
|
||||||
|
result += "😏检测到环境变量：" + str(osenv) + " = " + self.get_env(osenv) + " 将按照该路径为准\n"
|
||||||
|
pyabspath = self.get_env(osenv)
|
||||||
|
if os.path.exists(only_path):
|
||||||
|
with open(only_path, 'r+') as f:
|
||||||
|
py_data = f.read(2097152)
|
||||||
|
if py_data == pyabspath:
|
||||||
|
# 对旧版转换为json格式
|
||||||
|
j_data["py_path"] = pyabspath
|
||||||
|
f.seek(0, os.SEEK_SET)
|
||||||
|
result += "😏检测到旧版检测文件，自动转换新版\n"
|
||||||
|
f.writelines(json.dumps(j_data))
|
||||||
|
try:
|
||||||
|
j_data = json.loads(py_data)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
if j_data["py_path"] and j_data["py_path"] == pyabspath:
|
||||||
|
result += "😁脚本唯一性检测通过，继续运行!\n"
|
||||||
|
else:
|
||||||
|
result += "🙄检测到其他同类型的青龙日志分析脚本存在，拒绝运行!\n"
|
||||||
|
load_send()
|
||||||
|
send(pyname, result)
|
||||||
|
exit(0)
|
||||||
|
else:
|
||||||
|
with open(only_path, "w") as f:
|
||||||
|
j_data["py_path"] = pyabspath
|
||||||
|
f.writelines(json.dumps(j_data))
|
||||||
|
result += "🙄检测到第一次运行，已写入唯一性检测文件，如无特殊情况请勿删除\n"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def load_send():
|
||||||
|
global send
|
||||||
|
cur_path = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
sys.path.append(cur_path)
|
||||||
|
if os.path.exists(cur_path + "/notify.py"):
|
||||||
|
try:
|
||||||
|
from notify import send
|
||||||
|
except:
|
||||||
|
send = False
|
||||||
|
print("加载通知服务失败~")
|
||||||
|
else:
|
||||||
|
send = False
|
||||||
|
print("加载通知服务失败~")
|
||||||
270
jd_ql_log_scan.py
Normal file
270
jd_ql_log_scan.py
Normal file
@ -0,0 +1,270 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import traceback
|
||||||
|
from depend import Depend
|
||||||
|
|
||||||
|
'''
|
||||||
|
cron: 30 23 * * *
|
||||||
|
new Env('青龙日志分析 && 自动补全依赖');
|
||||||
|
########环境变量设置#########
|
||||||
|
|
||||||
|
## (非必填) 脚本唯一性检测，请在此处填写你想运行的脚本的绝对路径，其他脚本检测到路径与此变量不符将会停止运行
|
||||||
|
QL_LOG_SCAN_SCRIPT_PATH=
|
||||||
|
|
||||||
|
## (非必填)指定日志目录: 默认自动识别青龙目录，出现错误才需要手动指定日志目录
|
||||||
|
export QL_LOG_PATH="/ql/data/log/"
|
||||||
|
|
||||||
|
## (非必填)指定不扫描目录：多个请用逗号隔开
|
||||||
|
export QL_LOG_BLACK_DIR=""
|
||||||
|
|
||||||
|
## (非必填)指定不扫描日志文件：多个请用逗号隔开
|
||||||
|
export QL_LOG_BLACK_FILE=""
|
||||||
|
|
||||||
|
## (非必填)需要被扫描的最近n天的日志，0就是只分析当天的日志(最近24小时的日志)
|
||||||
|
export QL_LOG_SCAN_DEEPIN=0
|
||||||
|
|
||||||
|
## (非必填)是否尝试自动补齐日志报错里提示的依赖
|
||||||
|
export QL_LOG_AUTO_INSTALL_DEPEND=False
|
||||||
|
|
||||||
|
## (非必填)强制指定npm包管理器，有些青龙使用了pnpm而不是npm，注意鉴别
|
||||||
|
export QL_LOG_NPM="npm"
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
class QlLogScan(Depend):
|
||||||
|
def __init__(self):
|
||||||
|
self.pyname = os.path.basename(__file__).replace(".py", "")
|
||||||
|
print(self.only_check(self.pyname, os.path.abspath(__file__),"QL_LOG_SCAN_SCRIPT_PATH"))
|
||||||
|
self.ql_log_path = self.get_env("QL_LOG_PATH", self.get_ql_path() + "log/")
|
||||||
|
self.filter_dir_list = self.not2append(["^\.tmp$", "^update$", self.pyname + "$"],
|
||||||
|
self.str2list(self.get_env("QL_LOG_BLACK_DIR")))
|
||||||
|
self.filter_log_list = self.not2append(['task_error\.log', 'start\.log'],
|
||||||
|
self.str2list(self.get_env("QL_LOG_BLACK_FILE")))
|
||||||
|
self.history_scan_deepin = self.get_env("QL_LOG_SCAN_DEEPIN", "0")
|
||||||
|
self.auto_install_depend = self.get_env("QL_LOG_AUTO_INSTALL_DEPEND", False)
|
||||||
|
self.npm = self.get_env("QL_LOG_NPM", "npm")
|
||||||
|
self.log_stat = {
|
||||||
|
"all": 0,
|
||||||
|
"nodejs_err": 0,
|
||||||
|
"python_err": 0,
|
||||||
|
"err_dict": {},
|
||||||
|
"nodejs_depend": [],
|
||||||
|
"python_depend": [],
|
||||||
|
"readlog_err" :[]
|
||||||
|
}
|
||||||
|
self.LogNameHeadList = self.generateLogNameHeadList()
|
||||||
|
self.analysisLog()
|
||||||
|
self.showAnalysisLog()
|
||||||
|
if self.auto_install_depend:
|
||||||
|
self.auto_depend()
|
||||||
|
|
||||||
|
def generateLogNameHeadList(self):
|
||||||
|
scan_list = []
|
||||||
|
for i in range((self.history_scan_deepin + 1) * 24):
|
||||||
|
scan_list.append(time.strftime("%Y-%m-%d-%H", time.localtime((int(time.time()) - (3600 * i)))))
|
||||||
|
return scan_list
|
||||||
|
|
||||||
|
def analysisLog(self):
|
||||||
|
for path, dir_list, file_list in os.walk(self.ql_log_path):
|
||||||
|
dir_name = path.replace(self.ql_log_path, "")
|
||||||
|
if not self.re_filter_list(dir_name, self.filter_dir_list):
|
||||||
|
for file_name in file_list:
|
||||||
|
if not self.re_filter_list(file_name, self.filter_log_list) and re.search(r"(.*?).log$",
|
||||||
|
file_name) and file_name[
|
||||||
|
:13] in self.LogNameHeadList:
|
||||||
|
# 读取日志
|
||||||
|
log_file = open(os.path.join(path, file_name), "r")
|
||||||
|
try:
|
||||||
|
log_text = log_file.read(2097152)
|
||||||
|
log_file.close()
|
||||||
|
# 分析日志
|
||||||
|
nodejs_err_list = re.findall(r"Error\:(.*\s?)Require stack\:", log_text)
|
||||||
|
python_err_list = re.findall(
|
||||||
|
r"Traceback \(most recent call last\):([\n\s]+File[\s\S]*?, line [\d]+, in[\s\S]*?["
|
||||||
|
r"\s\S]*?\n[\s\S]*?\n)+(.*?)\n",
|
||||||
|
log_text)
|
||||||
|
if nodejs_err_list:
|
||||||
|
self.log_stat["nodejs_err"] += len(nodejs_err_list)
|
||||||
|
self.log_stat["err_dict"][dir_name] = []
|
||||||
|
for i in nodejs_err_list:
|
||||||
|
v = i.strip()
|
||||||
|
self.log_stat["err_dict"][dir_name].append({"type": "NodeJs", "log": v})
|
||||||
|
# 依赖缺失判断
|
||||||
|
miss_depend = re.search(r"Cannot find module '([a-zA-Z\d_-]+)'", v)
|
||||||
|
if miss_depend and miss_depend.group(1) not in self.log_stat["nodejs_depend"]:
|
||||||
|
self.log_stat["nodejs_depend"].append(miss_depend.group(1))
|
||||||
|
elif python_err_list:
|
||||||
|
self.log_stat["python_err"] += len(python_err_list)
|
||||||
|
self.log_stat["err_dict"][dir_name] = []
|
||||||
|
for i in python_err_list:
|
||||||
|
v = i[-1].strip()
|
||||||
|
self.log_stat["err_dict"][dir_name].append({"type": "Python", "log": v})
|
||||||
|
# 依赖缺失判断
|
||||||
|
miss_depend = re.search(r"ModuleNotFoundError: No module named \'([a-zA-Z0-9_-]+)\'", v)
|
||||||
|
if miss_depend and miss_depend.group(1) not in self.log_stat["python_depend"]:
|
||||||
|
self.log_stat["python_depend"].append(miss_depend.group(1))
|
||||||
|
self.log_stat["all"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
err_log = "读取日志" + str(os.path.join(path, file_name)) + "出现异常: " + str(e) + "\n"
|
||||||
|
self.log_stat["readlog_err"].append(err_log)
|
||||||
|
print(err_log)
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def format_log_date(text):
|
||||||
|
text = text.split("-")
|
||||||
|
return text[0] + "年" + text[1] + "月" + text[2] + "日" + text[3] + "时"
|
||||||
|
|
||||||
|
def showAnalysisLog(self):
|
||||||
|
len_nodejs_depend = len(self.log_stat["nodejs_depend"])
|
||||||
|
len_python_depend = len(self.log_stat["python_depend"])
|
||||||
|
# 展示分析结果
|
||||||
|
result = "📆分析 " + (
|
||||||
|
self.format_log_date(self.LogNameHeadList[0]) + " ~ " + self.format_log_date(
|
||||||
|
self.LogNameHeadList[-1]) if len(self.LogNameHeadList) != 1 else
|
||||||
|
self.LogNameHeadList[
|
||||||
|
0]) + " 的日志报告：\n"
|
||||||
|
if len(self.log_stat["readlog_err"]) != 0:
|
||||||
|
result += "🔍脚本在读取日志过程中，出现了" + str(len(self.log_stat["readlog_err"])) + "个异常，详细信息将在最后展示\n"
|
||||||
|
result += "✅正常运行脚本：" + str(self.log_stat["all"]) + " 次\n"
|
||||||
|
if self.log_stat["all"] != 0:
|
||||||
|
result += "⛔异常运行脚本：" + str(self.log_stat["nodejs_err"] + self.log_stat["python_err"]) + " 次，占比 " + str(
|
||||||
|
round(
|
||||||
|
(float(self.log_stat["nodejs_err"] + self.log_stat["python_err"]) / float(
|
||||||
|
self.log_stat["all"]) * 100),
|
||||||
|
2)) + " %\n"
|
||||||
|
result += "🧐其中:\n"
|
||||||
|
result += " 🕵️♂️Nodejs异常：" + str(self.log_stat["nodejs_err"]) + " 次，占比 " + str(
|
||||||
|
round((float(self.log_stat["nodejs_err"]) / float(self.log_stat["all"]) * 100), 2)) + " %\n"
|
||||||
|
result += " 🕵️♂️Python异常：" + str(self.log_stat["python_err"]) + " 次，占比 " + str(
|
||||||
|
round((float(self.log_stat["python_err"]) / float(self.log_stat["all"]) * 100), 2)) + " %\n"
|
||||||
|
if len_nodejs_depend > 0 or len_python_depend > 0:
|
||||||
|
result += "👮♂️依赖检测: " + (
|
||||||
|
"☢已开启自动补全依赖,将执行shell命令,请小心恶意脚本👿" if self.auto_install_depend else "❎未开启自动补全依赖，请手动补齐以下依赖🤗") + "\n"
|
||||||
|
if len_nodejs_depend > 0:
|
||||||
|
result += "👮♂️检测到缺失NodeJs依赖:\n"
|
||||||
|
result += str(self.log_stat["nodejs_depend"]) + "\n"
|
||||||
|
if len_python_depend > 0:
|
||||||
|
result += "👮♂️检测到缺失Python依赖:\n"
|
||||||
|
result += str(self.log_stat["python_depend"]) + "\n"
|
||||||
|
result += "💂♂️详细错误日志：\n\n"
|
||||||
|
|
||||||
|
for k, v in self.log_stat["err_dict"].items():
|
||||||
|
if v:
|
||||||
|
result += "🛑脚本：" + k + "：\n"
|
||||||
|
for i in v:
|
||||||
|
result += "- ⚠" + i["type"] + "错误：" + i["log"] + " \n\n\n"
|
||||||
|
if len(self.log_stat["readlog_err"]) != 0:
|
||||||
|
result += "👷♀️读取日志异常日志：\n\n"
|
||||||
|
for i in self.log_stat["readlog_err"]:
|
||||||
|
result += "⚠" + i + "\n"
|
||||||
|
send("🐲青龙日志分析", result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def auto_depend(self):
|
||||||
|
len_nodejs_depend = len(self.log_stat["nodejs_depend"])
|
||||||
|
len_python_depend = len(self.log_stat["python_depend"])
|
||||||
|
len_all_depend = len_nodejs_depend + len_python_depend
|
||||||
|
if len_nodejs_depend > 0:
|
||||||
|
for i in range(len_nodejs_depend):
|
||||||
|
shell_log = "🤖检测是否安装NodeJs依赖: " + self.log_stat["nodejs_depend"][i] + "\n"
|
||||||
|
check_result = self.check_depend(self.log_stat["nodejs_depend"][i], "nodejs")
|
||||||
|
if check_result:
|
||||||
|
shell_log += "📦" + str(check_result) + "已安装, 跳过安装\n"
|
||||||
|
else:
|
||||||
|
shell_log += "⚙当前正在自动安装NodeJs依赖: " + self.log_stat["nodejs_depend"][i] + "\n"
|
||||||
|
install_result = self.install_depend(self.log_stat["nodejs_depend"][i], "nodejs")
|
||||||
|
shell_log += "🔨执行命令: " + install_result[0] + "\n"
|
||||||
|
if install_result[2] != '':
|
||||||
|
shell_log += "⛔出错了: \n" + install_result[2] + "\n\n"
|
||||||
|
elif install_result[1] != '':
|
||||||
|
shell_log += "✅执行完成: \n" + install_result[1] + "\n\n"
|
||||||
|
send("🐲青龙自动安装依赖(" + str(i + 1) + "/" + str(len_all_depend) + ")", shell_log)
|
||||||
|
if len_python_depend > 0:
|
||||||
|
for i in range(len_python_depend):
|
||||||
|
shell_log = "🤖检测是否安装Python依赖: " + self.log_stat["python_depend"][i] + "\n"
|
||||||
|
check_result = self.check_depend(self.log_stat["python_depend"][i], "python")
|
||||||
|
if check_result:
|
||||||
|
shell_log += "📦" + str(check_result) + "已安装, 跳过安装\n"
|
||||||
|
else:
|
||||||
|
shell_log += "⚙当前正在自动安装Python依赖: " + self.log_stat["python_depend"][i] + "\n"
|
||||||
|
install_result = self.install_depend(self.log_stat["python_depend"][i], "python")
|
||||||
|
shell_log += "🔨执行命令: " + install_result[0] + "\n"
|
||||||
|
if install_result[2] != '':
|
||||||
|
shell_log += "⛔出错了: \n" + install_result[2] + "\n\n"
|
||||||
|
elif install_result[1] != '':
|
||||||
|
shell_log += "✅执行完成: \n" + install_result[1] + "\n\n"
|
||||||
|
send("🐲青龙自动安装依赖(" + str(i + 1 + len_nodejs_depend) + "/" + str(len_all_depend) + ")", shell_log)
|
||||||
|
|
||||||
|
def install_depend(self, package, package_type):
|
||||||
|
package = package.replace("+", "\+")
|
||||||
|
if package_type == "nodejs":
|
||||||
|
install_exec = 'cd /ql/ && ' + self.npm + ' install ' + package
|
||||||
|
elif package_type == "python":
|
||||||
|
install_exec = 'pip3 install ' + package
|
||||||
|
elif package_type == "docker":
|
||||||
|
install_exec = 'apk update && apk add ' + package
|
||||||
|
|
||||||
|
if install_exec:
|
||||||
|
install = subprocess.run(install_exec, shell=True, capture_output=True, text=True)
|
||||||
|
install_log = install.stdout
|
||||||
|
install_err = install.stderr
|
||||||
|
return install_exec, install_log, install_err
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def check_depend(self, package, package_type):
|
||||||
|
package = package.replace("+", "\+")
|
||||||
|
if package_type == "nodejs":
|
||||||
|
list_exec = 'cd /ql/ && ' + self.npm + ' list|grep ' + package
|
||||||
|
list_log = subprocess.run(list_exec, shell=True, capture_output=True, text=True).stdout
|
||||||
|
npm_re = re.search(r"[\s]" + package + "@[\d.]+", list_log)
|
||||||
|
pnpm_re = re.search(r"^" + package + " [\d.]+", list_log)
|
||||||
|
if npm_re:
|
||||||
|
return npm_re.group()
|
||||||
|
elif pnpm_re:
|
||||||
|
return pnpm_re.group()
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
elif package_type == "python":
|
||||||
|
list_exec = 'pip3 list|grep ' + package
|
||||||
|
list_log = subprocess.run(list_exec, shell=True, capture_output=True, text=True).stdout
|
||||||
|
pip_re = re.search(package + "[ ]+[\d.]+", list_log)
|
||||||
|
if pip_re:
|
||||||
|
return pip_re.group()
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
elif package_type == "docker":
|
||||||
|
list_exec = 'apk list|grep ' + package
|
||||||
|
list_log = subprocess.run(list_exec, shell=True, capture_output=True, text=True).stdout
|
||||||
|
docker_re = re.search(package + "-[\d.]+", list_log)
|
||||||
|
if docker_re:
|
||||||
|
return docker_re.group()
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def load_send():
|
||||||
|
global send
|
||||||
|
cur_path = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
sys.path.append(cur_path)
|
||||||
|
if os.path.exists(cur_path + "/notify.py"):
|
||||||
|
try:
|
||||||
|
from notify import send
|
||||||
|
except:
|
||||||
|
send = False
|
||||||
|
print("加载通知服务失败~")
|
||||||
|
else:
|
||||||
|
send = False
|
||||||
|
print("加载通知服务失败~")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
load_send()
|
||||||
|
ql = QlLogScan()
|
||||||
Loading…
Reference in New Issue
Block a user