从 “被动救火” 到 “主动预判”：用 NeuralProphet 搭建运维数据 AI 预测体系

conda create -n forecast pythnotallow=3.12

conda activate forecast

# 基于 PyTorch 的时间序列预测工具，用来训练模型并进行预测
pip install neuralprophet
# 用于处理Excel的库，演示用数据保存在Excel
pip install openpyxl
# 用于构建网页应用界面
pip install streamlit

cd /forecast
# 直接编辑，在保存时会生成文件
vim history_data.py

import numpy as np
import pandas as pd

# ============================
# 参数配置
# ============================
start_time = pd.Timestamp("2025-08-01 00:00:00")
end_time = pd.Timestamp("2025-08-31 23:59:59")
rng = pd.date_range(start=start_time, end=end_time, freq="10min")

periods = len(rng)
np.random.seed(42)  # 保证可复现

# ============================
# 辅助函数：日内波动模式
# ============================
def daily_pattern(minutes, amplitude=1.0, phase_shift=0):
    radians = (minutes / (24 * 60)) * 2 * np.pi
    return 0.5 + 0.5 * np.sin(radians - 0.3 + phase_shift)

minutes_from_start = (np.arange(periods) * 10) % (24 * 60)
weekday = rng.weekday  # 0=周一
hours = rng.hour

# ============================
# CPU
# ============================
cpu_base = 10 + 5 * (np.random.rand(periods) - 0.5)
cpu_daily = 25 * daily_pattern(minutes_from_start)
cpu = cpu_base + cpu_daily * np.where(weekday < 5, 1.0, 0.7)
spike_mask = (np.random.rand(periods) < 0.005)  # 偶发高峰
cpu[spike_mask] += np.random.uniform(20, 60, size=spike_mask.sum())

# ============================
# 内存
# ============================
mem_base = 50 + 10 * np.sin((np.arange(periods) / periods) * 2 * np.pi)
mem_noise = np.random.normal(0, 2.0, size=periods)
mem_weekday_effect = np.where(weekday < 5, 1.02, 0.99)
memory_used = mem_base * mem_weekday_effect + mem_noise
mem_spike_mask = (np.random.rand(periods) < 0.002)
memory_used[mem_spike_mask] += np.random.uniform(5, 20, size=mem_spike_mask.sum())
memory_used = np.clip(memory_used, 10, 99.5)

# ============================
# 磁盘使用率
# ============================
disk_start, disk_end = 60.0, 62.0
disk_trend = np.linspace(disk_start, disk_end, periods)
disk_noise = np.random.normal(0, 0.05, size=periods)
disk_used = np.clip(disk_trend + disk_noise, 20, 99.9)

# ============================
# 网络流量
# ============================
net_in_base = 5 * daily_pattern(minutes_from_start) + 1.0
net_out_base = 3 * daily_pattern(minutes_from_start) + 0.5
lunch_mask = ((hours >= 11) & (hours <= 13))
net_in = net_in_base * (1.2 * np.where(weekday < 5, 1, 0.9))
net_out = net_out_base * (1.15 * np.where(weekday < 5, 1, 0.9))
net_in[lunch_mask] *= 1.3
net_out[lunch_mask] *= 1.2
burst_mask = (np.random.rand(periods) < 0.003)
net_in[burst_mask] += np.random.uniform(20, 200, size=burst_mask.sum())
net_out[burst_mask] += np.random.uniform(5, 80, size=burst_mask.sum())

# ============================
# 组合 DataFrame
# ============================
df_aug = pd.DataFrame({
    "ds": rng,
    "cpu": np.round(np.clip(cpu, 0, 100), 2),
    "memory": np.round(memory_used, 2),
    "disk": np.round(disk_used, 2),
    "net_in": np.round(net_in, 3),
    "net_out": np.round(net_out, 3)
})

# ============================
# 每小时统计（平均、最大、最小）
# ============================
df_hourly = df_aug.set_index("ds").resample("1h").agg(
    {
        "cpu": ["mean", "max", "min"],
        "memory": ["mean", "max", "min"],
        "disk": ["mean", "max", "min"],
        "net_in": ["mean", "max", "min"],
        "net_out": ["mean", "max", "min"],
    }
)
df_hourly.columns = ["_".join(col).strip() for col in df_hourly.columns.values]
df_hourly.reset_index(inplace=True)


# ============================
# 为每个指标创建单独的sheet（ds:日期，y:值）
# ============================
# CPU sheet
df_cpu = pd.DataFrame({
    "ds": rng,
    "y": cpu
})

# Memory sheet
df_memory = pd.DataFrame({
    "ds": rng,
    "y": memory_used
})

# Disk sheet
df_disk = pd.DataFrame({
    "ds": rng,
    "y": disk_used
})

# Network In sheet
df_net_in = pd.DataFrame({
    "ds": rng,
    "y": net_in
})

# Network Out sheet
df_net_out = pd.DataFrame({
    "ds": rng,
    "y": net_out
})

# ============================
# 导出到 Excel
# ============================
output_file = "server_metrics_2025_08.xlsx"
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
    df_aug.to_excel(writer, index=False, sheet_name="10min_data")
    df_hourly.to_excel(writer, index=False, sheet_name="hourly_stats")
    # 添加各个指标的单独sheet
    df_cpu.to_excel(writer, index=False, sheet_name="cpu")
    df_memory.to_excel(writer, index=False, sheet_name="memory")
    df_disk.to_excel(writer, index=False, sheet_name="disk")
    df_net_in.to_excel(writer, index=False, sheet_name="net_in")
    df_net_out.to_excel(writer, index=False, sheet_name="net_out")

print(f"✅ 数据已生成并导出到 {output_file}")

python history_data.py

cd /forecast
# 直接编辑，在保存时会生成文件
vim train_model.py

from neuralprophet import NeuralProphet
import pandas as pd
import warnings
from neuralprophet import save

# 忽略警告
warnings.filterwarnings('ignore')

# 1. 读取Excel数据
excel_path = 'server_metrics_2025_08.xlsx'
sheet_name = 'cpu'
# sheet_name = 'memory'
# sheet_name = 'disk'
# sheet_name = 'net_in'
# sheet_name = 'net_out'
df = pd.read_excel(excel_path, sheet_name=sheet_name)

# 只保留 ds 和 y 列
df = df[['ds', 'y']]

# 2. 初始化并训练模型
m = NeuralProphet(
    changepoints_range=0.8,         # 只在前80%历史数据中寻找变点，避免对最新数据过拟合
    trend_reg=1,                    # 适度正则化，防止趋势过度拟合 
    seasonality_reg=0.5,            # 适度正则化，防止季节性过拟合
    n_lags=144,                     # n_lags越大，可用训练样本越少
    ar_reg=0.7,                     # 轻微正则化自回归系数，防止过拟合
    n_forecasts=6,                  # 预测的步数，10分数间隔的数据生成6条
    collect_metrics={ "MAE": "MeanAbsoluteError", "MAPE": "MeanAbsolutePercentageError" },
)

# 3. 分割训练集和验证集
df_train, df_val = m.split_df(df, valid_p=0.2)

# 4. 训练模型
m.fit(df_train, validation_df=df_val, freq="10min")

# 5. 保存模型到本地目录
save(m, sheet_name + ".np")

cd /forecast
# 直接编辑，在保存时会生成文件
vim app.py

import streamlit as st
import pandas as pd
import plotly.graph_objects as go
from neuralprophet import load
import warnings
import numpy as np
from datetime import datetime, timedelta
import sys


def generate_forecast(df, sheet_name):
    """
    生成预测的独立函数
    按步数遍历，步数1取yhat1，步数2取yhat2...
    所有预测值都放到yhat1列中
    """
    try:
        # 在预测函数中仅保留'ds'和'y'列用于模型训练
        model_df = df[['ds', 'y']].copy()
        
        # 加载预训练模型
        model_path = f"{sheet_name}.np"
        
        model = load(model_path)
        
        # 生成预测
        future_df = model.make_future_dataframe(model_df, n_historic_predictions=False)
        forecast_result = model.predict(future_df, decompose=False)

        print(forecast_result)

        # 获取最后一个数据点
        last_data_point = model_df['ds'].max()

        # 处理预测结果
        forecast_long = []
        last_actual_idx = forecast_result['y'].last_valid_index() if 'y' in forecast_result.columns else -1
        start_idx = last_actual_idx + 1 if last_actual_idx + 1 < len(forecast_result) else 0

        # 生成预测步骤，按步数取对应yhat，统一放到yhat1列
        for step in range(1, FORECAST_STEPS + 1):
            yhat_col = f'yhat{step}'  # 步数1取yhat1，步数2取yhat2...
            
            current_idx = start_idx + (step - 1)
            if current_idx >= len(forecast_result):
                current_idx = len(forecast_result) - 1
                st.warning(f"预测数据不足，使用最后一条数据预测第{step}步")
            
            if current_idx < len(forecast_result):
                date = forecast_result.loc[current_idx, 'ds']
                
                if pd.notna(date) and yhat_col in forecast_result.columns and pd.notna(forecast_result.loc[current_idx, yhat_col]):
                    forecast_time = last_data_point + step * FORCE_TIME_INTERVAL
                    # 所有预测值都放到yhat1列
                    forecast_entry = {
                        'ds': forecast_time,
                        'yhat1': forecast_result.loc[current_idx, yhat_col],
                        'step': step
                    }
                    forecast_long.append(forecast_entry)
                else:
                    st.warning(f"第{step}步预测值缺失，使用历史平均值")
                    forecast_time = last_data_point + step * FORCE_TIME_INTERVAL
                    avg_value = model_df['y'].mean()
                    forecast_entry = {
                        'ds': forecast_time,
                        'yhat1': avg_value,
                        'step': step
                    }
                    forecast_long.append(forecast_entry)
            else:
                forecast_time = last_data_point + step * FORCE_TIME_INTERVAL
                avg_value = model_df['y'].mean()
                forecast_entry = {
                    'ds': forecast_time,
                    'yhat1': avg_value,
                    'step': step
                }
                forecast_long.append(forecast_entry)

        # 转换为DataFrame
        forecast_long = pd.DataFrame(forecast_long)
        forecast_merged = forecast_long.merge(df[['ds', 'y']], on='ds', how='left')
        
        return forecast_merged
        
    except Exception as e:
        st.error(f"预测出错: {str(e)}")
        st.info("可能的原因：模型文件不存在、数据格式不正确或模型训练不完整")
        return None

streamlit run app.py

# python-dotenv：可以使用load_dotenv从同目录的.env文件中读取配置
pip install python-dotenv openai

DEEPSEEK_API_KEY=sk-*********

import os
from pathlib import Path

# 项目根目录
ROOT_DIR = Path(__file__).parent

# Excel文件路径
EXCEL_PATH = os.path.join(ROOT_DIR, 'server_metrics_2025_08.xlsx')

# 资源类型列表
RESOURCE_TYPES = ['cpu', 'memory', 'disk', 'net_in', 'net_out']

# 资源类型中文映射
RESOURCE_TYPE_MAP = {
    'cpu': 'CPU使用率',
    'memory': '内存使用率',
    'disk': '磁盘使用率',
    'net_in': '网络入流量',
    'net_out': '网络出流量'
}

# 时间格式
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
DATE_FORMAT = '%Y-%m-%d'

import logging
import os
from datetime import datetime
from pathlib import Path

# 创建logs目录
log_dir = Path(__file__).parent / 'logs'
log_dir.mkdir(exist_ok=True)

# 配置日志格式
log_format = '%(asctime)s - %(levelname)s - %(message)s'
date_format = '%Y-%m-%d %H:%M:%S'

# 创建日志文件名（包含日期）
log_file = log_dir / f'chat_app_{datetime.now().strftime("%Y%m%d")}.log'

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format=log_format,
    datefmt=date_format,
    handlers=[
        logging.FileHandler(log_file, encoding='utf-8'),
        logging.StreamHandler()
    ]
)

# 创建logger实例
logger = logging.getLogger(__name__)

def initialize_openai_client():
    """初始化OpenAI客户端
    
    功能：
    - 从环境变量获取API密钥
    - 初始化DeepSeek API客户端
    - 配置API基础URL
    
    返回：
    - OpenAI客户端实例 或 None（如果初始化失败）
    """
    # 从环境变量获取API密钥
    api_key = os.getenv('DEEPSEEK_API_KEY')
    if not api_key:
        st.error("未找到DEEPSEEK_API_KEY环境变量")
        return None
    
    # 创建并返回配置好的客户端实例
    return OpenAI(
        api_key=api_key,
        base_url="https://api.deepseek.com/v1"  # DeepSeek API的基础URL
    )

def generate_prompt(start_dt, end_dt, resource_type=None):
    """生成提示词：支持时间点或时间范围。
    - 当 start_dt == end_dt 时，生成“在 X 时 ...”的时间点提示
    - 当不相等时，生成“在 X 至 Y 这段时间 ...”的时间范围提示
    """
    if not start_dt or not end_dt:
        return ""

    start_str = start_dt.strftime("%Y-%m-%d %H:%M:%S")
    end_str = end_dt.strftime("%Y-%m-%d %H:%M:%S")

    resource_name = RESOURCE_TYPE_MAP.get(resource_type, resource_type) if resource_type else None

    if start_dt == end_dt:
        if resource_name:
            return f"在{start_str} 时服务器的{resource_name}情况如何？请进行分析，指出异常与风险。"
        return f"在{start_str} 时服务器的整体运行情况如何？请进行分析，指出异常与风险。"
    else:
        if resource_name:
            return f"在{start_str} 至 {end_str} 这段时间内，服务器的{resource_name}情况如何？请进行分析，指出异常与风险。"
        return f"在{start_str} 至 {end_str} 这段时间内，服务器的整体运行情况如何？请进行分析，指出异常与风险。"

def filter_data(df, date, time=None, resource_type=None, end_date=None, end_time=None):
    """根据选择的日期时间范围和资源类型筛选数据"""
    if df is None:
        logger.warning("输入的DataFrame为空")
        return None
    
    logger.info(f"开始筛选数据，起始日期：{date}, 起始时间：{time or '00:00:00'}")
    logger.info(f"结束日期：{end_date or date}, 结束时间：{end_time or '23:59:59'}")
    logger.info(f"资源类型：{resource_type or '所有资源'}")
    
    # 构建开始日期时间
    start_datetime = pd.Timestamp.combine(date, time) if time else pd.Timestamp.combine(date, pd.Timestamp.min.time())
    
    # 构建结束日期时间
    if end_date and end_time:
        end_datetime = pd.Timestamp.combine(end_date, end_time)
    elif end_date:
        end_datetime = pd.Timestamp.combine(end_date, pd.Timestamp.max.time())
    else:
        end_datetime = pd.Timestamp.combine(date, pd.Timestamp.max.time())
    
    logger.info(f"筛选时间范围：{start_datetime} 至 {end_datetime}")
    
    # 筛选时间范围内的数据
    filtered_df = df[(df['ds'] >= start_datetime) & (df['ds'] <= end_datetime)]
    
    # 记录筛选结果
    logger.info(f"时间范围内的数据条数：{len(filtered_df)}")
    
    # 如果选择了资源类型，只返回相关列
    if resource_type and resource_type in df.columns:
        filtered_df = filtered_df[['ds', resource_type]]
        logger.info(f"已筛选资源类型：{resource_type}")
    
    # 记录最终结果
    logger.info(f"最终筛选结果数据条数：{len(filtered_df)}")
    if not filtered_df.empty:
        logger.info(f"数据时间范围：{filtered_df['ds'].min()} 至 {filtered_df['ds'].max()}")
        if resource_type:
            stats = filtered_df[resource_type].describe()
            logger.info(f"资源统计信息：\n{stats}")
    else:
        logger.warning("筛选结果为空")
    
    return filtered_df

streamlit run chat_app.py