智能运维新实践：Prometheus 与 DeepSeek 联动实现告警自动分析 原创

docker run --name prometheus -d -p 127.0.0.1:9090:9090 prom/prometheus

LOGIN_COUNT = Counter(
 'user_login_total', # 指标名称 (Metric Name)
 '用户登录总次数及状态统计', # 指标描述 (Help Text)
 ['user_type', 'login_status', 'ip_region'] # 标签/维度 (Labels/Dimensions)
 )

pip install prometheus-client

from prometheus_client import start_http_server, Counter
 import random
 import time

 # 定义一个计数器类型的指标（累计值，适合记录次数）
 # 标签说明：
 # - user_type: 用户类型（普通用户/管理员）
 # - login_status: 登录状态（成功/失败）
 # - ip_region: IP所属地区（模拟分布式用户场景）
 LOGIN_COUNT = Counter(
 'user_login_total', # 指标名（遵循Prometheus命名规范：小写+下划线）
 '用户登录总次数及状态统计', # 指标描述
 ['user_type', 'login_status', 'ip_region'] # 标签（用于多维度筛选）
 )

 def simulate_login():
 """模拟用户登录行为（每3秒产生一次登录事件）"""
 # 模拟用户类型（70%普通用户，30%管理员）
 user_type = random.choices(['normal', 'admin'], weights=[0.7, 0.3])[0]
 
 # 模拟登录状态（90%成功，10%失败）
 login_status = random.choices(['success', 'failed'], weights=[0.9, 0.1])[0]
 
 # 模拟IP地区（国内主要城市）
 ip_region = random.choice(['beijing', 'shanghai', 'guangzhou', 'shenzhen', 'hangzhou'])
 
 # 记录一次登录事件（计数器+1）
 LOGIN_COUNT.labels(
 user_type=user_type,
 login_status=login_status,
 ip_reginotallow=ip_region
 ).inc() # 每次调用+1

 if __name__ == '__main__':
 # 启动HTTP服务，暴露指标在9091端口
 start_http_server(9091)
 print("用户登录监控Exporter运行在 http://localhost:9091/metrics")
 
 # 持续模拟登录事件
 while True:
 simulate_login()
 time.sleep(3) # 每3秒模拟一次登录

def simulate_login():
 # 随机生成用户类型（70%普通用户，30%管理员）
 user_type = random.choices(['normal', 'admin'], weights=[0.7, 0.3])[0]
 
 # 随机生成登录结果（90%成功，10%失败）
 login_status = random.choices(['success', 'failed'], weights=[0.9, 0.1])[0]
 
 # 随机选择IP地区（国内主要城市）
 ip_region = random.choice(['beijing', 'shanghai', 'guangzhou', 'shenzhen', 'hangzhou'])
 
 # 记录登录事件
 LOGIN_COUNT.labels(
 user_type=user_type,
 login_status=login_status,
 ip_reginotallow=ip_region
 ).inc() # 计数器+1

LOGIN_COUNT = Counter(
 'user_login_total', # 指标名称
 '用户登录总次数及状态统计', # 指标描述
 ['user_type', 'login_status', 'ip_region'] # 三维度标签
 )

if __name__ == '__main__':
 start_http_server(9091) # 启动指标暴露服务
 print("服务运行在 http://localhost:9091/metrics")
 
 while True:
 simulate_login()
 time.sleep(3)

python ~/docker/prometheus/custom_exporter.py

Exporter运行在 http://localhost:9091/metrics

# HELP python_gc_objects_collected_total Objects collected during gc
 # TYPE python_gc_objects_collected_total counter
 python_gc_objects_collected_total{generatinotallow="0"} 255.0
 python_gc_objects_collected_total{generatinotallow="1"} 118.0
 python_gc_objects_collected_total{generatinotallow="2"} 0.0
 # HELP python_gc_objects_uncollectable_total Uncollectable object found during GC
 # TYPE python_gc_objects_uncollectable_total counter
 python_gc_objects_uncollectable_total{generatinotallow="0"} 0.0
 python_gc_objects_uncollectable_total{generatinotallow="1"} 0.0
 python_gc_objects_uncollectable_total{generatinotallow="2"} 0.0
 # HELP python_gc_collections_total Number of times this generation was collected
 # TYPE python_gc_collections_total counter
 python_gc_collections_total{generatinotallow="0"} 40.0
 python_gc_collections_total{generatinotallow="1"} 3.0
 python_gc_collections_total{generatinotallow="2"} 0.0
 # HELP python_info Python platform information
 # TYPE python_info gauge
 python_info{implementatinotallow="CPython",major="3",minor="9",patchlevel="18",versinotallow="3.9.18"} 1.0
 # HELP my_custom_metric 自定义测试指标
 # TYPE my_custom_metric gauge
 my_custom_metric{label1="test",label2="normal"} 1.3487897665970805

global:
 scrape_interval: 5s # 每15秒拉取一次数据

 scrape_configs:
 # 监控Prometheus自身
 - job_name: 'prometheus'
 static_configs:
 - targets: ['localhost:9090']

 # 监控本地的自定义Exporter（关键配置）
 - job_name: 'custom_exporter'
 static_configs:
 # 这里的IP不是localhost，而是你本地机器的实际IP（容器内访问宿主机的地址）
 # 如何获取：在本地终端执行 `ifconfig` 或 `ip addr` 查看（例如192.168.x.x）
 - targets: ['192.168.0.8:9091'] # 你的本地IP + Exporter端口

docker stop prometheus
 docker rm prometheus

docker run --name prometheus -d -p 127.0.0.1:9090:9090 -v /Users/cuihao/docker/prometheus:/etc/prometheus prom/prometheus --config.file=/etc/prometheus/prometheus.yml

user_login_total

groups:
 - name: login_anomaly_rules # 规则组名称
 rules:
 # 规则1：5分钟内登录失败率超过20%（可能是攻击或系统故障）
 - alert: HighLoginFailureRate
 expr: |
 sum(rate(user_login_total{login_status="failed"}[5m])) 
 / 
 sum(rate(user_login_total[5m])) 
 > 0.2 # 失败率>20%
 for: 1m # 持续1分钟触发告警
 labels:
 severity: critical # 告警级别（紧急）
 annotations:
 summary: "登录失败率过高"
 description: "过去5分钟登录失败率{{ $value | humanizePercentage }}，可能存在异常登录行为"

 # 规则2：管理员账号1分钟内登录失败次数>5次（可能被暴力破解）
 - alert: AdminLoginFailureSpike
 expr: |
 sum(rate(user_login_total{user_type="admin", login_status="failed"}[1m])) 
 > 5 # 1分钟内失败>5次
 for: 30s # 持续30秒触发
 labels:
 severity: warning
 annotations:
 summary: "管理员登录失败次数突增"
 description: "管理员账号1分钟内登录失败{{ $value }}次，可能存在暴力破解风险"

global:
 scrape_interval: 15s

 rule_files:
 - "alert_rules.yml" # 加载刚才创建的告警规则文件

 scrape_configs:
 - job_name: 'custom_exporter'
 static_configs:
 - targets: ['192.168.0.16:9091'] # 你的Exporter地址

docker restart prometheus

# 先创建Alertmanager配置目录
 mkdir -p /Users/cuihao/docker/prometheus/alertmanager

 # 启动容器（映射9093端口，挂载配置目录）
 docker run --name alertmanager -d -p 127.0.0.1:9093:9093 -v /Users/cuihao/docker/prometheus/alertmanager:/etc/alertmanager prom/alertmanager

route:
 group_by: ['alertname'] # 按告警名称分组
 group_wait: 10s # 组内等待10秒再发送
 group_interval: 10s # 组内间隔10秒发送
 repeat_interval: 1h # 重复告警间隔1小时
 receiver: 'webhook_handler' # 转发到名为webhook_handler的接收者

 receivers:
 - name: 'webhook_handler'
 webhook_configs:
 - url: 'http://192.168.0.8:5008/alert' # 你的处理服务地址（本地5000端口）
 send_resolved: true # 告警解决后也发送通知

docker restart alertmanager

docker stop prometheus alertmanager

docker network create prometheus-network

# 将Prometheus加入网络
 docker network connect prometheus-network prometheus

 # 将Alertmanager加入网络
 docker network connect prometheus-network alertmanager

docker start prometheus alertmanager

# 进入Prometheus容器
 docker exec -it prometheus /bin/sh

 # 测试DNS解析（若容器内有nslookup）
 nslookup alertmanager

 # 若没有nslookup，用ping测试（能ping通说明解析正常）
 ping alertmanager

alerting:
 alertmanagers:
 - static_configs:
 - targets:
 - 'alertmanager:9093'

docker restart prometheus

from flask import Flask, request, jsonify
 import os
 import sys
 import datetime
 import json
 from openai import OpenAI
 from dotenv import load_dotenv # 用于加载环境变量

 # 初始化Flask应用
 app = Flask(__name__)

 # 加载环境变量（从.env文件读取DEEPSEEK_API_KEY）
 try:
 # 尝试加载环境变量，并添加日志
 load_result = load_dotenv() # 默认读取当前目录的.env文件
 script_dir = os.path.dirname(os.path.abspath(__file__))
 env_path = os.path.join(script_dir, ".env")
 
 # 配置日志和报告路径（使用脚本所在目录）
 error_log = os.path.join(script_dir, "error.log") # 错误日志路径
 report_dir = os.path.join(script_dir, "alert_reports") # 分析报告保存目录
 
 # 记录环境变量加载情况
 with open(error_log, "a", encoding="utf-8") as f:
 if load_result:
 f.write(f"[{datetime.datetime.now()}] 成功加载环境变量文件: {env_path}\n")
 else:
 f.write(f"[{datetime.datetime.now()}] 未找到环境变量文件，使用系统环境变量: {env_path}\n")
 
 # 确保报告目录存在
 os.makedirs(report_dir, exist_ok=True)
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 报告目录准备就绪: {report_dir}\n")
 except Exception as e:
 # 记录初始化错误
 error_msg = f"初始化文件系统时出错: {str(e)}"
 print(f"[{datetime.datetime.now()}] {error_msg}") # 控制台也输出，方便调试
 # 尝试写入错误日志
 try:
 with open("error.log", "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] {error_msg}\n")
 except:
 pass
 raise # 抛出错误，终止启动

 def init_deepseek_client():
 """初始化DeepSeek客户端（兼容OpenAI SDK）"""
 try:
 # 验证环境变量
 api_key = os.environ.get("DEEPSEEK_API_KEY")
 with open(error_log, "a", encoding="utf-8") as f:
 if api_key:
 f.write(f"[{datetime.datetime.now()}] 成功获取DEEPSEEK_API_KEY (部分隐藏): {api_key[:4]}****\n")
 else:
 f.write(f"[{datetime.datetime.now()}] 未在环境变量中找到DEEPSEEK_API_KEY\n")
 
 if not api_key:
 raise ValueError("未在环境变量中找到DEEPSEEK_API_KEY，请检查.env文件")
 
 # 初始化客户端（DeepSeek兼容OpenAI SDK格式）
 client = OpenAI(
 api_key=api_key,
 base_url="https://api.deepseek.com/v1" # DeepSeek API基础地址
 )
 
 # 验证客户端连接
 try:
 # 发送一个简单的测试请求验证连接
 response = client.chat.completions.create(
 model="deepseek-chat",
 messages=[{"role": "user", "content": "test"}],
 max_tokens=1
 )
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] DeepSeek客户端初始化成功，API连接正常\n")
 except Exception as e:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] DeepSeek API连接测试失败: {str(e)}\n")
 raise
 
 return client
 except Exception as e:
 # 记录初始化错误
 error_msg = f"DeepSeek客户端初始化失败：{str(e)}"
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] {error_msg}\n")
 raise # 抛出错误，避免服务启动后无法调用API

 # 初始化DeepSeek客户端（服务启动时执行）
 try:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 开始初始化DeepSeek客户端...\n")
 client = init_deepseek_client()
 except Exception as e:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 初始化DeepSeek客户端失败，服务无法启动：{str(e)}\n")
 raise


 def call_deepseek_analysis(alert_info):
 """调用DeepSeek分析告警信息，返回分析结果和报告路径"""
 try:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 开始处理告警分析...\n")
 
 # 1. 验证并提取告警关键信息
 if not isinstance(alert_info, dict):
 raise ValueError(f"告警信息格式错误，预期字典类型，实际为: {type(alert_info)}")
 
 # 验证必要字段
 required_fields = ["labels", "annotations"]
 for field in required_fields:
 if field not in alert_info:
 raise ValueError(f"告警信息缺少必要字段: {field}")
 
 alert_name = alert_info["labels"].get("alertname", "未知告警")
 severity = alert_info["labels"].get("severity", "未知级别")
 description = alert_info["annotations"].get("description", "无描述")
 start_time = alert_info.get("startsAt", datetime.datetime.now().isoformat())
 labels = alert_info["labels"]
 
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 提取到告警信息 - 名称: {alert_name}, 级别: {severity}\n")

 # 2. 构造提示词（系统角色+用户输入）
 system_prompt = """

""" user_prompt = f"""

- 告警名称：{alert_name}
 - 严重级别：{severity}
 - 告警描述：{description}
 - 发生时间：{start_time}
 - 关联标签：{json.dumps(labels, ensure_ascii=False)} # 包含用户类型、地区等维度
 
 请基于以上信息分析并生成报告。
 """

 # 3. 调用DeepSeek模型
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 开始调用DeepSeek API进行分析...\n")
 
 response = client.chat.completions.create(
 model="deepseek-chat", # 使用的模型名称
 messages=[
 {"role": "system", "content": system_prompt},
 {"role": "user", "content": user_prompt}
 ],
 stream=False,
 max_tokens=1000 # 限制最大返回长度
 )

 # 4. 提取分析结果
 if not response.choices or len(response.choices) == 0:
 raise ValueError("DeepSeek API返回结果为空")
 
 analysis_content = response.choices[0].message.content
 
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 成功获取DeepSeek分析结果，长度: {len(analysis_content)}\n")

 # 5. 生成分析报告文件
 timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
 report_filename = f"analysis_{alert_name}_{timestamp}.txt".replace(" ", "_")
 report_path = os.path.join(report_dir, report_filename)
 
 with open(report_path, "w", encoding="utf-8") as f:
 f.write(f"=== 告警分析报告 ===\n")
 f.write(f"生成时间：{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
 f.write(f"告警名称：{alert_name}\n")
 f.write(f"严重级别：{severity}\n")
 f.write(f"发生时间：{start_time}\n\n")
 f.write("=== 分析内容 ===\n")
 f.write(analysis_content)
 
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 分析报告已保存至: {report_path}\n")

 return analysis_content, report_path

 except Exception as e:
 # 记录调用错误
 error_msg = f"DeepSeek分析调用失败：{str(e)}"
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] {error_msg}\n")
 return f"分析失败：{str(e)}", None


 @app.route('/alert', methods=['POST'])
 def handle_alert():
 """接收Alertmanager的Webhook告警，调用分析并返回结果"""
 try:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 收到新的告警请求\n")
 
 # 1. 解析Alertmanager发送的JSON数据
 try:
 alert_data = request.json
 except Exception as e:
 raise ValueError(f"解析JSON数据失败: {str(e)}")
 
 if not alert_data:
 return jsonify({"status": "failed", "reason": "未收到有效数据"}), 400
 
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 成功解析告警数据，包含字段: {', '.join(alert_data.keys())}\n")

 # 2. 提取第一个触发的告警
 if not alert_data.get("alerts"):
 return jsonify({"status": "failed", "reason": "告警数据中无alerts字段"}), 400
 
 if not isinstance(alert_data["alerts"], list) or len(alert_data["alerts"]) == 0:
 return jsonify({"status": "failed", "reason": "alerts字段不是有效的列表或为空"}), 400
 
 first_alert = alert_data["alerts"][0]
 alert_name = first_alert['labels'].get('alertname', '未知告警')
 print(f"[{datetime.datetime.now()}] 收到告警：{alert_name}") # 控制台输出
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 处理第一个告警: {alert_name}\n")

 # 3. 调用DeepSeek分析
 analysis_content, report_path = call_deepseek_analysis(first_alert)

 # 4. 返回结果
 return jsonify({
 "status": "success",
 "analysis": analysis_content,
 "report_path": report_path
 })

 except Exception as e:
 # 记录请求处理错误
 error_msg = f"告警处理失败：{str(e)}"
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] {error_msg}\n")
 return jsonify({"status": "failed", "reason": str(e)}), 500

 # 添加一个健康检查接口，方便测试服务是否正常运行
 @app.route('/health', methods=['GET'])
 def health_check():
 try:
 return jsonify({
 "status": "healthy",
 "timestamp": datetime.datetime.now().isoformat(),
 "services": {
 "deepseek": "initialized" if 'client' in globals() else "not initialized",
 "report_dir": report_dir,
 "report_dir_writable": os.access(report_dir, os.W_OK)
 }
 })
 except Exception as e:
 return jsonify({
 "status": "unhealthy",
 "reason": str(e)
 }), 500


 if __name__ == '__main__':
 try:
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] 开始启动alert_handler服务...\n")
 f.write(f"[{datetime.datetime.now()}] 服务将监听端口: 5008\n")
 
 # 启动服务（监听所有IP的5008端口）
 app.run(host='0.0.0.0', port=5008, debug=True)
 except Exception as e:
 error_msg = f"服务启动失败：{str(e)}"
 print(f"[{datetime.datetime.now()}] {error_msg}")
 with open(error_log, "a", encoding="utf-8") as f:
 f.write(f"[{datetime.datetime.now()}] {error_msg}\n")
 sys.exit(1)

python alert_handler.py

curl -X POST "http://localhost:5008/alert" \
 -H "Content-Type: application/json" \
 -d '{
 "version": "4",
 "groupKey": "{}:{alertname=\"HighLoginFailureRate\"}",
 "status": "firing",
 "receiver": "webhook_handler",
 "groupLabels": {
 "alertname": "HighLoginFailureRate"
 },
 "commonLabels": {
 "alertname": "HighLoginFailureRate",
 "severity": "critical"
 },
 "commonAnnotations": {
 "description": "登录失败率过高，当前失败率为30%，超过阈值20%",
 "summary": "登录失败率异常升高"
 },
 "externalURL": "http://localhost:9093",
 "alerts": [
 {
 "status": "firing",
 "labels": {
 "alertname": "HighLoginFailureRate",
 "severity": "critical"
 },
 "annotations": {
 "description": "登录失败率过高，当前失败率为30%，超过阈值20%",
 "summary": "登录失败率异常升高"
 },
 "startsAt": "2025-08-14T08:30:00.000Z",
 "endsAt": "0001-01-01T00:00:00Z",
 "generatorURL": "http://localhost:9090/graph?g0.expr=sum(rate(user_login_total%7Blogin_status%3D%22failed%22%7D%5B5m%5D))+%2F+sum(rate(user_login_total%5B5m%5D))+%3E+0.2&g0.tab=1"
 }
 ]
 }'