12040900

models_processing/model_prediction_lightgbm/.ipynb_checkpoints/Dockerfile-checkpoint

@@ -1,16 +0,0 @@
-# 使用官方 Python 镜像作为基础镜像
-FROM 192.168.1.36:5000/python:3.9
-ENV LANG=en_US.UTF-8
-# 设置工作目录
-WORKDIR /app
-
-# 复制当前目录内容到工作目录
-COPY . /app
-
-# 安装依赖项（如果有）
-RUN set -ex \
-    && pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-# 设置 ENTRYPOINT 为 Python 脚本
-ENTRYPOINT ["python", "model_prediction_lightgbm.py"]
-# 执行 Python 脚本
-CMD []

models_processing/model_prediction_lightgbm/.ipynb_checkpoints/model_prediction_lightgbm-checkpoint.py

@@ -1,99 +0,0 @@
-import lightgbm as lgb
-import argparse
-import pandas as pd 
-import numpy as np
-from pymongo import MongoClient
-import pickle
-from flask import Flask,request,jsonify
-from waitress import serve
-import time
-import logging
-import traceback
-
-app = Flask('model_prediction_lightgbm——service')
-
-
-def get_data_from_mongo(args):
-    mongodb_connection,mongodb_database,mongodb_read_table,timeBegin,timeEnd = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_read_table'],args['timeBegin'],args['timeEnd']
-    client = MongoClient(mongodb_connection)
-    # 选择数据库（如果数据库不存在，MongoDB 会自动创建）
-    db = client[mongodb_database]
-    collection = db[mongodb_read_table]  # 集合名称
-    query = {"dateTime": {"$gte": timeBegin, "$lte": timeEnd}}
-    cursor = collection.find(query)
-    data = list(cursor)
-    df = pd.DataFrame(data)
-    # 4. 删除 _id 字段（可选）
-    if '_id' in df.columns:
-        df = df.drop(columns=['_id'])
-    client.close()
-    return df
-    
-
-def insert_data_into_mongo(res_df,args):
-    mongodb_connection,mongodb_database,mongodb_write_table = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_write_table']
-    client = MongoClient(mongodb_connection)
-    db = client[mongodb_database]
-    if mongodb_write_table in db.list_collection_names():
-        db[mongodb_write_table].drop()
-        print(f"Collection '{mongodb_write_table} already exist, deleted successfully!")
-    collection = db[mongodb_write_table]  # 集合名称
-    # 将 DataFrame 转为字典格式
-    data_dict = res_df.to_dict("records")  # 每一行作为一个字典
-    # 插入到 MongoDB
-    collection.insert_many(data_dict)
-    print("data inserted successfully!")
-    
-
-def model_prediction(df,args):
-    mongodb_connection,mongodb_database,mongodb_model_table,model_name = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_model_table'],args['model_name']
-    client = MongoClient(mongodb_connection)
-    db = client[mongodb_database]
-    collection = db[mongodb_model_table]
-    model_data = collection.find_one({"model_name": model_name})
-    if model_data is not None:
-        model_binary = model_data['model']  # 确保这个字段是存储模型的二进制数据
-        # 反序列化模型 
-        model = pickle.loads(model_binary)
-        df['predict'] = model.predict(df[model.feature_name()])
-        print("model predict result  successfully!")
-    return df
-
-
-@app.route('/model_prediction_lightgbm', methods=['POST'])
-def model_prediction_lightgbm():
-    # 获取程序开始时间  
-    start_time = time.time()  
-    result = {}
-    success = 0
-    print("Program starts execution!")
-    try:
-        args = request.values.to_dict()
-        print('args',args)
-        logger.info(args)
-        power_df = get_data_from_mongo(args)
-        model = model_prediction(power_df,args)
-        insert_data_into_mongo(model,args)
-        success = 1
-    except Exception as e:
-        my_exception = traceback.format_exc()
-        my_exception.replace("\n","\t")
-        result['msg'] = my_exception
-    end_time = time.time() 
-   
-    result['success'] = success
-    result['args'] = args
-    result['start_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time))
-    result['end_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end_time))
-    print("Program execution ends!")
-    return result
-
-
-if __name__=="__main__":  
-    print("Program starts execution!")
-    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    logger = logging.getLogger("model_prediction_lightgbm log")
-    from waitress import serve
-    serve(app, host="0.0.0.0", port=10090)
-    print("server start!")
-    

models_processing/model_prediction_lightgbm/.ipynb_checkpoints/requirements-checkpoint.txt

@@ -1,7 +0,0 @@
-pymongo==4.9.1
-pandas==2.1.4
-numpy==1.26.4
-lightgbm==4.3.0
-scikit-learn==1.4.2
-Flask==3.0.3
-waitress==3.0.0

models_processing/model_prediction_lightgbm/.ipynb_checkpoints/test-checkpoint.ipynb

@@ -1,490 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "660798fa-cf08-4cfd-b553-c6d750d7e7ad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import algo_test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "ff42670b-301d-4ee1-8da2-6a658fc5f286",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'_id': ObjectId('66ebba82a3b7dbe82538e34a'), 'name': 'Alice', 'age': 25, 'city': 'Beijing'}\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 使用 subprocess 执行 .py 文件\n",
-    "result = subprocess.run(['python', 'algo_test.py'], capture_output=True, text=True)\n",
-    "\n",
-    "# 打印输出\n",
-    "print(result.stdout)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "b47d615a-5953-43c0-9289-387d5c823a7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip freeze > requirements.txt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "752c3ab3-cdef-47b0-9d9a-805348d30d6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "3.9.19 (main, May  6 2024, 20:12:36) [MSC v.1916 64 bit (AMD64)]\n"
-     ]
-    }
-   ],
-   "source": [
-    "parser.add_argument('--mongodb_connection', type=str, required=True, help=\"mongodb连接\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_database', type=str, required=True, help=\"mongodb数据库名\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_read_table', type=str, required=True, help=\"mongodb数据库读取表\")# tmp_one\n",
-    "parser.add_argument('--mongodb_model_table', type=str, required=True, help=\"mongodb读取模型集合\")# tmp_one\n",
-    "parser.add_argument('--model_name', type=str, required=True, help=\"保存模型名\")# tmp_one\n",
-    "#出参\n",
-    "parser.add_argument('--mongodb_write_table', type=str, required=True, help=\"mongodb数据库存储集合\")# tmp_one"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "id": "acc6aa2a-0f1d-43f3-985d-cd37c4a5ac6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Output:\n",
-      "Program starts execution!\n",
-      "args: Namespace(numerical_features=['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'], categorical_features=[], label='实际功率', learning_rate=0.01, num_leaves=31, min_data_in_leaf=21, mongodb_connection='mongodb://root:Jydl3377@192.168.1.33:27017', mongodb_database='algorithm_platform', mongodb_read_table='J00260NWP_records', mongodb_write_table='models', model_name='lightgbm_model')\n",
-      "df:************                         _id               C_TIME  ...   NWP-湿度    NWP-气压\n",
-      "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  ...  23.4918  892.3327\n",
-      "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  ...  24.0436  892.3809\n",
-      "\n",
-      "[2 rows x 34 columns]\n",
-      "Starting training...\n",
-      "[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000981 seconds.\n",
-      "You can set `force_col_wise=true` to remove the overhead.\n",
-      "[LightGBM] [Info] Total Bins 6885\n",
-      "[LightGBM] [Info] Number of data points in the train set: 7770, number of used features: 27\n",
-      "[LightGBM] [Info] Start training from score 47.616663\n",
-      "Training completed!\n",
-      "Collection 'models already exist, deleted successfully!\n",
-      "model inserted successfully!\n",
-      "Program execution ends!\n",
-      "\n",
-      "Error:\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 构建命令\n",
-    "command = [\n",
-    "    \"python\", \"model_prediction.py\",\n",
-    "    \"--mongodb_connection\", \"mongodb://root:Jydl3377@192.168.1.33:27017\",\n",
-    "    \"--mongodb_database\", \"algorithm_platform\",\n",
-    "    \"--mongodb_read_table\", \"J00260NWP_records\",\n",
-    "    \"--mongodb_model_table\", \"models\",\n",
-    "    \"--model_name\",\"lightgbm_model\",\n",
-    "    \"--mongodb_write_table\", \"J00260NWP_records_predict\",\n",
-    "]\n",
-    "\n",
-    "# 使用 subprocess.run 来执行命令\n",
-    "result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
-    "\n",
-    "# 输出结果和错误信息\n",
-    "print(\"Output:\")\n",
-    "print(result.stdout)\n",
-    "print(\"Error:\")\n",
-    "print(result.stderr)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "14a9feb4-5625-45d7-8e0f-6a8dea8c9f24",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run lightgbm_model.py   --numerical_features \"2024-05-01\"     --categorical_features \"2024-09-20\"   --label \"J00260\"     --learning_rate \"C_TIME\"    --num_leaves \"mongodb://root:Jydl3377@192.168.1.33:27017\"     --mongondb_database \"algorithm_platform\"     --mongondb_read_table \"J00260NWP_machines\"     --mongondb_write_table \"J00260NWP_records\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c0d791f8-12d8-4ab9-b561-b474e2a739f0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run lightgbm_model.py --numerical_features ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'])  --categorical_features  ','.join([])  --label  \"实际功率\" --learning_rate  \"0.01\" --num_leaves \"31\"  --min_data_in_leaf  \"21\"  --mongodb_connection  \"mongodb://root:Jydl3377@192.168.1.33:27017\" --mongodb_database  \"algorithm_platform\" --mongodb_read_table  \"J00260NWP_records\" --mongodb_write_table  \"models\" --model_name \"lightgbm_model\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "9ca17599-e453-479c-a9b3-b1d15f2d54de",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lightgbm.basic.Booster at 0x16ab35646d0>"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import lightgbm as lgb\n",
-    "import argparse\n",
-    "import pandas as pd \n",
-    "import numpy as np\n",
-    "from pymongo import MongoClient\n",
-    "import pickle\n",
-    "client = MongoClient(\"mongodb://root:Jydl3377@192.168.1.33:27017\")\n",
-    "db = client[\"algorithm_platform\"]\n",
-    "collection = db[\"models\"]\n",
-    "model_data = collection.find_one({\"model_name\": \"lightgbm_model\"})\n",
-    "if model_data is not None:\n",
-    "    # 假设模型存储在字段 'model' 中\n",
-    "    model_binary = model_data['model']  # 确保这个字段是存储模型的二进制数据\n",
-    "\n",
-    "    # 反序列化模型\n",
-    "    model = pickle.loads(model_binary)\n",
-    "model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "ff940fec-55fd-4cbd-99ef-5564c6632efb",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>_id</th>\n",
-       "      <th>C_TIME</th>\n",
-       "      <th>理论功率</th>\n",
-       "      <th>第1个设备轮毂风速</th>\n",
-       "      <th>实际功率</th>\n",
-       "      <th>实时环境数据</th>\n",
-       "      <th>预测功率</th>\n",
-       "      <th>NWP预测总辐射</th>\n",
-       "      <th>NWP-10米预测风速</th>\n",
-       "      <th>NWP-10米预测风向</th>\n",
-       "      <th>...</th>\n",
-       "      <th>NWP-低云量</th>\n",
-       "      <th>NWP-中云量</th>\n",
-       "      <th>NWP-高云量</th>\n",
-       "      <th>NWP-总云量</th>\n",
-       "      <th>NWP-晴空总辐射</th>\n",
-       "      <th>NWP-法相直辐射</th>\n",
-       "      <th>NWP-总降水强度</th>\n",
-       "      <th>NWP-温度</th>\n",
-       "      <th>NWP-湿度</th>\n",
-       "      <th>NWP-气压</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>66ecc86f1075358d129a1d56</td>\n",
-       "      <td>2024-05-02 22:30:00</td>\n",
-       "      <td>130.05</td>\n",
-       "      <td>9.63</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>92.52</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.6477</td>\n",
-       "      <td>202.7945</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0173</td>\n",
-       "      <td>20.2918</td>\n",
-       "      <td>20.3927</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>14.0818</td>\n",
-       "      <td>23.4918</td>\n",
-       "      <td>892.3327</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>66ecc86f1075358d129a1d57</td>\n",
-       "      <td>2024-05-02 22:45:00</td>\n",
-       "      <td>130.51</td>\n",
-       "      <td>9.72</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>93.24</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.9228</td>\n",
-       "      <td>202.6482</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0191</td>\n",
-       "      <td>20.3009</td>\n",
-       "      <td>20.3864</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>13.8264</td>\n",
-       "      <td>24.0436</td>\n",
-       "      <td>892.3809</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>2 rows × 34 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                        _id               C_TIME    理论功率  第1个设备轮毂风速    实际功率  \\\n",
-       "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  130.05       9.63  104.76   \n",
-       "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  130.51       9.72  104.76   \n",
-       "\n",
-       "   实时环境数据   预测功率  NWP预测总辐射  NWP-10米预测风速  NWP-10米预测风向  ...  NWP-低云量  NWP-中云量  \\\n",
-       "0   -0.99  92.52       0.0       3.6477     202.7945  ...      0.0   0.0173   \n",
-       "1   -0.99  93.24       0.0       3.9228     202.6482  ...      0.0   0.0191   \n",
-       "\n",
-       "   NWP-高云量  NWP-总云量  NWP-晴空总辐射  NWP-法相直辐射  NWP-总降水强度   NWP-温度   NWP-湿度  \\\n",
-       "0  20.2918  20.3927        0.0        0.0        0.0  14.0818  23.4918   \n",
-       "1  20.3009  20.3864        0.0        0.0        0.0  13.8264  24.0436   \n",
-       "\n",
-       "     NWP-气压  \n",
-       "0  892.3327  \n",
-       "1  892.3809  \n",
-       "\n",
-       "[2 rows x 34 columns]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "\n",
-    "collection = db[\"J00260NWP_records\"]  # 集合名称\n",
-    "data_from_db = collection.find()  # 这会返回一个游标（cursor）\n",
-    "# 将游标转换为列表，并创建 pandas DataFrame\n",
-    "df = pd.DataFrame(list(data_from_db))\n",
-    "df.head(2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "7feaff6d-e41a-4bd7-b6fb-a63fc6900cca",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df['predict'] = model.predict(df[model.feature_name()])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "8a22f3e9-feff-438e-81e2-b5ceaa15ac48",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>_id</th>\n",
-       "      <th>C_TIME</th>\n",
-       "      <th>理论功率</th>\n",
-       "      <th>第1个设备轮毂风速</th>\n",
-       "      <th>实际功率</th>\n",
-       "      <th>实时环境数据</th>\n",
-       "      <th>预测功率</th>\n",
-       "      <th>NWP预测总辐射</th>\n",
-       "      <th>NWP-10米预测风速</th>\n",
-       "      <th>NWP-10米预测风向</th>\n",
-       "      <th>...</th>\n",
-       "      <th>NWP-中云量</th>\n",
-       "      <th>NWP-高云量</th>\n",
-       "      <th>NWP-总云量</th>\n",
-       "      <th>NWP-晴空总辐射</th>\n",
-       "      <th>NWP-法相直辐射</th>\n",
-       "      <th>NWP-总降水强度</th>\n",
-       "      <th>NWP-温度</th>\n",
-       "      <th>NWP-湿度</th>\n",
-       "      <th>NWP-气压</th>\n",
-       "      <th>predict</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>66ecc86f1075358d129a1d56</td>\n",
-       "      <td>2024-05-02 22:30:00</td>\n",
-       "      <td>130.05</td>\n",
-       "      <td>9.63</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>92.52</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.6477</td>\n",
-       "      <td>202.7945</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0173</td>\n",
-       "      <td>20.2918</td>\n",
-       "      <td>20.3927</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>14.0818</td>\n",
-       "      <td>23.4918</td>\n",
-       "      <td>892.3327</td>\n",
-       "      <td>77.728147</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>66ecc86f1075358d129a1d57</td>\n",
-       "      <td>2024-05-02 22:45:00</td>\n",
-       "      <td>130.51</td>\n",
-       "      <td>9.72</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>93.24</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.9228</td>\n",
-       "      <td>202.6482</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0191</td>\n",
-       "      <td>20.3009</td>\n",
-       "      <td>20.3864</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>13.8264</td>\n",
-       "      <td>24.0436</td>\n",
-       "      <td>892.3809</td>\n",
-       "      <td>84.568199</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>2 rows × 35 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                        _id               C_TIME    理论功率  第1个设备轮毂风速    实际功率  \\\n",
-       "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  130.05       9.63  104.76   \n",
-       "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  130.51       9.72  104.76   \n",
-       "\n",
-       "   实时环境数据   预测功率  NWP预测总辐射  NWP-10米预测风速  NWP-10米预测风向  ...  NWP-中云量  NWP-高云量  \\\n",
-       "0   -0.99  92.52       0.0       3.6477     202.7945  ...   0.0173  20.2918   \n",
-       "1   -0.99  93.24       0.0       3.9228     202.6482  ...   0.0191  20.3009   \n",
-       "\n",
-       "   NWP-总云量  NWP-晴空总辐射  NWP-法相直辐射  NWP-总降水强度   NWP-温度   NWP-湿度    NWP-气压  \\\n",
-       "0  20.3927        0.0        0.0        0.0  14.0818  23.4918  892.3327   \n",
-       "1  20.3864        0.0        0.0        0.0  13.8264  24.0436  892.3809   \n",
-       "\n",
-       "     predict  \n",
-       "0  77.728147  \n",
-       "1  84.568199  \n",
-       "\n",
-       "[2 rows x 35 columns]"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df.head(2)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python (panddlets)",
-   "language": "python",
-   "name": "panddlets"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.19"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

models_processing/model_prediction_lightgbm/Dockerfile

@@ -1,16 +0,0 @@
-# 使用官方 Python 镜像作为基础镜像
-FROM 192.168.1.36:5000/python:3.9
-ENV LANG=en_US.UTF-8
-# 设置工作目录
-WORKDIR /app
-
-# 复制当前目录内容到工作目录
-COPY . /app
-
-# 安装依赖项（如果有）
-RUN set -ex \
-    && pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-# 设置 ENTRYPOINT 为 Python 脚本
-ENTRYPOINT ["python", "model_prediction_lightgbm.py"]
-# 执行 Python 脚本
-CMD []

models_processing/model_prediction_lightgbm/requirements.txt

@@ -1,7 +0,0 @@
-pymongo==4.9.1
-pandas==2.1.4
-numpy==1.26.4
-lightgbm==4.3.0
-scikit-learn==1.4.2
-Flask==3.0.3
-waitress==3.0.0

models_processing/model_prediction_lightgbm/test.ipynb

@@ -1,537 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "660798fa-cf08-4cfd-b553-c6d750d7e7ad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import algo_test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "ff42670b-301d-4ee1-8da2-6a658fc5f286",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'_id': ObjectId('66ebba82a3b7dbe82538e34a'), 'name': 'Alice', 'age': 25, 'city': 'Beijing'}\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 使用 subprocess 执行 .py 文件\n",
-    "result = subprocess.run(['python', 'algo_test.py'], capture_output=True, text=True)\n",
-    "\n",
-    "# 打印输出\n",
-    "print(result.stdout)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "b47d615a-5953-43c0-9289-387d5c823a7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip freeze > requirements.txt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "752c3ab3-cdef-47b0-9d9a-805348d30d6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "3.9.19 (main, May  6 2024, 20:12:36) [MSC v.1916 64 bit (AMD64)]\n"
-     ]
-    }
-   ],
-   "source": [
-    "parser.add_argument('--mongodb_connection', type=str, required=True, help=\"mongodb连接\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_database', type=str, required=True, help=\"mongodb数据库名\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_read_table', type=str, required=True, help=\"mongodb数据库读取表\")# tmp_one\n",
-    "parser.add_argument('--mongodb_model_table', type=str, required=True, help=\"mongodb读取模型集合\")# tmp_one\n",
-    "parser.add_argument('--model_name', type=str, required=True, help=\"保存模型名\")# tmp_one\n",
-    "#出参\n",
-    "parser.add_argument('--mongodb_write_table', type=str, required=True, help=\"mongodb数据库存储集合\")# tmp_one"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "acc6aa2a-0f1d-43f3-985d-cd37c4a5ac6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Output:\n",
-      "Program starts execution!\n",
-      "args: Namespace(mongodb_connection='mongodb://root:Jydl3377@192.168.1.33:27017', mongodb_database='algorithm_platform', mongodb_read_table='J00260NWP_records', mongodb_model_table='models', model_name='lightgbm_model', mongodb_write_table='J00260NWP_records_predict')\n",
-      "model predict result  successfully!\n",
-      "data inserted successfully!\n",
-      "Program execution ends!\n",
-      "\n",
-      "Error:\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 构建命令\n",
-    "command = [\n",
-    "    \"python\", \"model_prediction.py\",\n",
-    "    \"--mongodb_connection\", \"mongodb://root:Jydl3377@192.168.1.33:27017\",\n",
-    "    \"--mongodb_database\", \"algorithm_platform\",\n",
-    "    \"--mongodb_read_table\", \"J00260NWP_records\",\n",
-    "    \"--mongodb_model_table\", \"models\",\n",
-    "    \"--model_name\",\"lightgbm_model\",\n",
-    "    \"--mongodb_write_table\", \"J00260NWP_records_predict\",\n",
-    "]\n",
-    "\n",
-    "# 使用 subprocess.run 来执行命令\n",
-    "result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
-    "\n",
-    "# 输出结果和错误信息\n",
-    "print(\"Output:\")\n",
-    "print(result.stdout)\n",
-    "print(\"Error:\")\n",
-    "print(result.stderr)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "14a9feb4-5625-45d7-8e0f-6a8dea8c9f24",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run model_prediction_lightgbm  --mongodb_connection \"mongodb://root:Jydl3377@192.168.1.33:27017\"     --mongodb_database \"algorithm_platform\"   --mongodb_read_table \"J00260NWP_records\"     --mongodb_model_table \"models\"    --model_name \"lightgbm_model\"     --mongodb_database \"algorithm_platform\"     --mongodb_write_table \"J00260NWP_records_predict\"      "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c0d791f8-12d8-4ab9-b561-b474e2a739f0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run lightgbm_model.py --numerical_features ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'])  --categorical_features  ','.join([])  --label  \"实际功率\" --learning_rate  \"0.01\" --num_leaves \"31\"  --min_data_in_leaf  \"21\"  --mongodb_connection  \"mongodb://root:Jydl3377@192.168.1.33:27017\" --mongodb_database  \"algorithm_platform\" --mongodb_read_table  \"J00260NWP_records\" --mongodb_write_table  \"models\" --model_name \"lightgbm_model\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6d13761d-3ba7-441c-903d-db293ab23223",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-11-21 15:58:04,711 - waitress - INFO - Serving on http://0.0.0.0:10090\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-11-21 16:00:49,195 - model_prediction_lightgbm log - INFO - {'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics', 'mongodb_model_table': 'models', 'mongodb_write_table': 'j00629_copy_statistics_predict', 'model_name': 'lightgbm_model', 'timeBegin': '2024-09-01 00:00:00', 'timeEnd': '2024-09-04 23:45:00'}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics', 'mongodb_model_table': 'models', 'mongodb_write_table': 'j00629_copy_statistics_predict', 'model_name': 'lightgbm_model', 'timeBegin': '2024-09-01 00:00:00', 'timeEnd': '2024-09-04 23:45:00'}\n",
-      "model predict result  successfully!\n",
-      "data inserted successfully!\n",
-      "Program execution ends!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-11-21 16:42:24,465 - model_prediction_lightgbm log - INFO - {'formulaType': 'DAY_SHORT_ACCURACY', 'cap': '225', 'province': 'E46', 'electricType': 'E1', 'stationCode': 'J00629', 'col_time': 'dateTime', 'col_rp': \"'realPower'\", 'col_pp': 'predict', 'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics_predict', 'mongodb_write_table': 'j00629_copy_statistics_predict_accuracy'}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'formulaType': 'DAY_SHORT_ACCURACY', 'cap': '225', 'province': 'E46', 'electricType': 'E1', 'stationCode': 'J00629', 'col_time': 'dateTime', 'col_rp': \"'realPower'\", 'col_pp': 'predict', 'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics_predict', 'mongodb_write_table': 'j00629_copy_statistics_predict_accuracy'}\n",
-      "Program execution ends!\n"
-     ]
-    }
-   ],
-   "source": [
-    " %run model_prediction_lightgbm.py"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "9ca17599-e453-479c-a9b3-b1d15f2d54de",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lightgbm.basic.Booster at 0x16ab35646d0>"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import lightgbm as lgb\n",
-    "import argparse\n",
-    "import pandas as pd \n",
-    "import numpy as np\n",
-    "from pymongo import MongoClient\n",
-    "import pickle\n",
-    "client = MongoClient(\"mongodb://root:Jydl3377@192.168.1.33:27017\")\n",
-    "db = client[\"algorithm_platform\"]\n",
-    "collection = db[\"models\"]\n",
-    "model_data = collection.find_one({\"model_name\": \"lightgbm_model\"})\n",
-    "if model_data is not None:\n",
-    "    # 假设模型存储在字段 'model' 中\n",
-    "    model_binary = model_data['model']  # 确保这个字段是存储模型的二进制数据\n",
-    "\n",
-    "    # 反序列化模型\n",
-    "    model = pickle.loads(model_binary)\n",
-    "model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "ff940fec-55fd-4cbd-99ef-5564c6632efb",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>_id</th>\n",
-       "      <th>C_TIME</th>\n",
-       "      <th>理论功率</th>\n",
-       "      <th>第1个设备轮毂风速</th>\n",
-       "      <th>实际功率</th>\n",
-       "      <th>实时环境数据</th>\n",
-       "      <th>预测功率</th>\n",
-       "      <th>NWP预测总辐射</th>\n",
-       "      <th>NWP-10米预测风速</th>\n",
-       "      <th>NWP-10米预测风向</th>\n",
-       "      <th>...</th>\n",
-       "      <th>NWP-中云量</th>\n",
-       "      <th>NWP-高云量</th>\n",
-       "      <th>NWP-总云量</th>\n",
-       "      <th>NWP-晴空总辐射</th>\n",
-       "      <th>NWP-法相直辐射</th>\n",
-       "      <th>NWP-总降水强度</th>\n",
-       "      <th>NWP-温度</th>\n",
-       "      <th>NWP-湿度</th>\n",
-       "      <th>NWP-气压</th>\n",
-       "      <th>predict</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>66ecc86f1075358d129a1d56</td>\n",
-       "      <td>2024-05-02 22:30:00</td>\n",
-       "      <td>130.05</td>\n",
-       "      <td>9.63</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>92.52</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.6477</td>\n",
-       "      <td>202.7945</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0173</td>\n",
-       "      <td>20.2918</td>\n",
-       "      <td>20.3927</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>14.0818</td>\n",
-       "      <td>23.4918</td>\n",
-       "      <td>892.3327</td>\n",
-       "      <td>77.728147</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>66ecc86f1075358d129a1d57</td>\n",
-       "      <td>2024-05-02 22:45:00</td>\n",
-       "      <td>130.51</td>\n",
-       "      <td>9.72</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>93.24</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.9228</td>\n",
-       "      <td>202.6482</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0191</td>\n",
-       "      <td>20.3009</td>\n",
-       "      <td>20.3864</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>13.8264</td>\n",
-       "      <td>24.0436</td>\n",
-       "      <td>892.3809</td>\n",
-       "      <td>84.568199</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>2 rows × 35 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                        _id               C_TIME    理论功率  第1个设备轮毂风速    实际功率  \\\n",
-       "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  130.05       9.63  104.76   \n",
-       "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  130.51       9.72  104.76   \n",
-       "\n",
-       "   实时环境数据   预测功率  NWP预测总辐射  NWP-10米预测风速  NWP-10米预测风向  ...  NWP-中云量  NWP-高云量  \\\n",
-       "0   -0.99  92.52       0.0       3.6477     202.7945  ...   0.0173  20.2918   \n",
-       "1   -0.99  93.24       0.0       3.9228     202.6482  ...   0.0191  20.3009   \n",
-       "\n",
-       "   NWP-总云量  NWP-晴空总辐射  NWP-法相直辐射  NWP-总降水强度   NWP-温度   NWP-湿度    NWP-气压  \\\n",
-       "0  20.3927        0.0        0.0        0.0  14.0818  23.4918  892.3327   \n",
-       "1  20.3864        0.0        0.0        0.0  13.8264  24.0436  892.3809   \n",
-       "\n",
-       "     predict  \n",
-       "0  77.728147  \n",
-       "1  84.568199  \n",
-       "\n",
-       "[2 rows x 35 columns]"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "\n",
-    "collection = db[\"J00260NWP_records_predict\"]  # 集合名称\n",
-    "data_from_db = collection.find()  # 这会返回一个游标（cursor）\n",
-    "# 将游标转换为列表，并创建 pandas DataFrame\n",
-    "df = pd.DataFrame(list(data_from_db))\n",
-    "df.head(2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "7feaff6d-e41a-4bd7-b6fb-a63fc6900cca",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df['predict'] = model.predict(df[model.feature_name()])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "8a22f3e9-feff-438e-81e2-b5ceaa15ac48",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>_id</th>\n",
-       "      <th>C_TIME</th>\n",
-       "      <th>理论功率</th>\n",
-       "      <th>第1个设备轮毂风速</th>\n",
-       "      <th>实际功率</th>\n",
-       "      <th>实时环境数据</th>\n",
-       "      <th>预测功率</th>\n",
-       "      <th>NWP预测总辐射</th>\n",
-       "      <th>NWP-10米预测风速</th>\n",
-       "      <th>NWP-10米预测风向</th>\n",
-       "      <th>...</th>\n",
-       "      <th>NWP-中云量</th>\n",
-       "      <th>NWP-高云量</th>\n",
-       "      <th>NWP-总云量</th>\n",
-       "      <th>NWP-晴空总辐射</th>\n",
-       "      <th>NWP-法相直辐射</th>\n",
-       "      <th>NWP-总降水强度</th>\n",
-       "      <th>NWP-温度</th>\n",
-       "      <th>NWP-湿度</th>\n",
-       "      <th>NWP-气压</th>\n",
-       "      <th>predict</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>66ecc86f1075358d129a1d56</td>\n",
-       "      <td>2024-05-02 22:30:00</td>\n",
-       "      <td>130.05</td>\n",
-       "      <td>9.63</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>92.52</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.6477</td>\n",
-       "      <td>202.7945</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0173</td>\n",
-       "      <td>20.2918</td>\n",
-       "      <td>20.3927</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>14.0818</td>\n",
-       "      <td>23.4918</td>\n",
-       "      <td>892.3327</td>\n",
-       "      <td>77.728147</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>66ecc86f1075358d129a1d57</td>\n",
-       "      <td>2024-05-02 22:45:00</td>\n",
-       "      <td>130.51</td>\n",
-       "      <td>9.72</td>\n",
-       "      <td>104.76</td>\n",
-       "      <td>-0.99</td>\n",
-       "      <td>93.24</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>3.9228</td>\n",
-       "      <td>202.6482</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.0191</td>\n",
-       "      <td>20.3009</td>\n",
-       "      <td>20.3864</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>13.8264</td>\n",
-       "      <td>24.0436</td>\n",
-       "      <td>892.3809</td>\n",
-       "      <td>84.568199</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>2 rows × 35 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                        _id               C_TIME    理论功率  第1个设备轮毂风速    实际功率  \\\n",
-       "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  130.05       9.63  104.76   \n",
-       "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  130.51       9.72  104.76   \n",
-       "\n",
-       "   实时环境数据   预测功率  NWP预测总辐射  NWP-10米预测风速  NWP-10米预测风向  ...  NWP-中云量  NWP-高云量  \\\n",
-       "0   -0.99  92.52       0.0       3.6477     202.7945  ...   0.0173  20.2918   \n",
-       "1   -0.99  93.24       0.0       3.9228     202.6482  ...   0.0191  20.3009   \n",
-       "\n",
-       "   NWP-总云量  NWP-晴空总辐射  NWP-法相直辐射  NWP-总降水强度   NWP-温度   NWP-湿度    NWP-气压  \\\n",
-       "0  20.3927        0.0        0.0        0.0  14.0818  23.4918  892.3327   \n",
-       "1  20.3864        0.0        0.0        0.0  13.8264  24.0436  892.3809   \n",
-       "\n",
-       "     predict  \n",
-       "0  77.728147  \n",
-       "1  84.568199  \n",
-       "\n",
-       "[2 rows x 35 columns]"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df.head(2)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python (panddlets)",
-   "language": "python",
-   "name": "panddlets"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.19"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

models_processing/model_training_lighgbm/.ipynb_checkpoints/Dockerfile-checkpoint

@@ -1,16 +0,0 @@
-# 使用官方 Python 镜像作为基础镜像
-FROM 192.168.1.36:5000/python:3.9
-ENV LANG=en_US.UTF-8
-# 设置工作目录
-WORKDIR /app
-
-# 复制当前目录内容到工作目录
-COPY . /app
-
-# 安装依赖项（如果有）
-RUN set -ex \
-    && pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-# 设置 ENTRYPOINT 为 Python 脚本
-ENTRYPOINT ["python", "model_training_lightgbm.py"]
-# 执行 Python 脚本
-CMD []

models_processing/model_training_lighgbm/.ipynb_checkpoints/model_training_lightgbm-checkpoint.py

@@ -1,130 +0,0 @@
-import lightgbm as lgb
-import argparse
-import pandas as pd 
-import numpy as np
-from pymongo import MongoClient
-import pickle
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import mean_squared_error,mean_absolute_error,r2_score
-from flask import Flask,request,jsonify
-from waitress import serve
-import time
-import logging
-import traceback
-
-app = Flask('model_training_lightgbm——service')
-def get_data_from_mongo(args):
-     mongodb_connection,mongodb_database,mongodb_read_table,timeBegin,timeEnd = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_read_table'],args['timeBegin'],args['timeEnd']
-    client = MongoClient(mongodb_connection)
-    # 选择数据库（如果数据库不存在，MongoDB 会自动创建）
-    db = client[mongodb_database]
-    collection = db[mongodb_read_table]  # 集合名称
-    query = {"dateTime": {"$gte": timeBegin, "$lte": timeEnd}}
-    cursor = collection.find(query)
-    data = list(cursor)
-    df = pd.DataFrame(data)
-    # 4. 删除 _id 字段（可选）
-    if '_id' in df.columns:
-        df = df.drop(columns=['_id'])
-    client.close()
-    return df
-    
-
-def insert_model_into_mongo(model_data,args):
-    mongodb_connection,mongodb_database,mongodb_write_table = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_write_table']
-    client = MongoClient(mongodb_connection)
-    db = client[mongodb_database]
-    if mongodb_write_table in db.list_collection_names():
-        db[mongodb_write_table].drop()
-        print(f"Collection '{mongodb_write_table} already exist, deleted successfully!")
-    collection = db[mongodb_write_table]  # 集合名称
-    collection.insert_one(model_data)
-    print("model inserted successfully!")
-
-
-def build_model(df,args):
-    np.random.seed(42)
-    #lightgbm预测下
-    numerical_features,categorical_features,label,model_name,num_boost_round,model_params = str_to_list(args['numerical_features']),str_to_list(args['categorical_features']),args['label'],args['model_name'],int(args['num_boost_round']),eval(args['model_params'])
-    features = numerical_features+categorical_features
-    print("features:************",features)
-    # 拆分数据为训练集和测试集
-    X_train, X_test, y_train, y_test = train_test_split(df[features], df[label], test_size=0.2, random_state=42)
-    # 创建LightGBM数据集
-    lgb_train = lgb.Dataset(X_train, y_train,categorical_feature=categorical_features)
-    lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
-    # 设置参数
-    params = {
-        'objective': 'regression',
-        'metric': 'rmse',
-        'boosting_type': 'gbdt',
-        'verbose':1
-    }
-
-    merged_param = params | model_params
-    # 训练模型
-    print('Starting training...')
-    gbm = lgb.train(merged_param,
-                    lgb_train,
-                    num_boost_round=num_boost_round,
-                    valid_sets=[lgb_train, lgb_eval],
-                    )
-    y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
-
-    # 评估
-    mse = mean_squared_error(y_test, y_pred)
-    rmse = np.sqrt(mse)
-    mae = mean_absolute_error(y_test, y_pred)
-    print(f'The test rmse is: {rmse},"The test mae is:"{mae}')
-    
-    # 序列化模型
-    model_bytes = pickle.dumps(gbm)
-    model_data = {
-                'model_name': model_name,
-                'model': model_bytes,     #将模型字节流存入数据库
-        }
-    print('Training completed!')
-    return model_data
-
-def str_to_list(arg):
-    if arg == '':
-        return []
-    else:
-        return arg.split(',')
-
-@app.route('/model_training_lightgbm', methods=['POST'])
-def model_training_lightgbm():
-    # 获取程序开始时间  
-    start_time = time.time()  
-    result = {}
-    success = 0
-    print("Program starts execution!")
-    try:
-        args = request.values.to_dict()
-        print('args',args)
-        logger.info(args)
-        power_df = get_data_from_mongo(args)
-        model = build_model(power_df,args)
-        insert_model_into_mongo(model,args)
-        success = 1
-    except Exception as e:
-        my_exception = traceback.format_exc()
-        my_exception.replace("\n","\t")
-        result['msg'] = my_exception
-    end_time = time.time() 
-   
-    result['success'] = success
-    result['args'] = args
-    result['start_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time))
-    result['end_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end_time))
-    print("Program execution ends!")
-    return result
-
-
-if __name__=="__main__":  
-    print("Program starts execution!")
-    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    logger = logging.getLogger("model_training_lightgbm log")
-    from waitress import serve
-    serve(app, host="0.0.0.0", port=10089)
-    print("server start!")

models_processing/model_training_lighgbm/.ipynb_checkpoints/requirements-checkpoint.txt

@@ -1,7 +0,0 @@
-pymongo==4.9.1
-pandas==2.1.4
-numpy==1.26.4
-lightgbm==4.3.0
-scikit-learn==1.4.2
-Flask==3.0.3
-waitress==3.0.0

models_processing/model_training_lighgbm/.ipynb_checkpoints/test-checkpoint.ipynb

@@ -1,179 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "660798fa-cf08-4cfd-b553-c6d750d7e7ad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import algo_test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "ff42670b-301d-4ee1-8da2-6a658fc5f286",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'_id': ObjectId('66ebba82a3b7dbe82538e34a'), 'name': 'Alice', 'age': 25, 'city': 'Beijing'}\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 使用 subprocess 执行 .py 文件\n",
-    "result = subprocess.run(['python', 'algo_test.py'], capture_output=True, text=True)\n",
-    "\n",
-    "# 打印输出\n",
-    "print(result.stdout)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "b47d615a-5953-43c0-9289-387d5c823a7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip freeze > requirements.txt"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "752c3ab3-cdef-47b0-9d9a-805348d30d6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "3.9.19 (main, May  6 2024, 20:12:36) [MSC v.1916 64 bit (AMD64)]\n"
-     ]
-    }
-   ],
-   "source": [
-    "parser.add_argument('--numerical_features', type=str, required=True, help=\"数值型特征\") \n",
-    "parser.add_argument('--categorical_features', type=str, required=True, help=\"类别型特征\")\n",
-    "parser.add_argument('--learning_rate', type=float, required=True, help=\"学习率\")\n",
-    "parser.add_argument('--num_leaves', type=int, required=True, help=\"叶子节点数\")\n",
-    "parser.add_argument('--min_data_in_leaf', type=int, required=True, help=\"叶子节点最小数据量\")# mongodb://root:Jydl3377@192.168.1.33:27017\n",
-    "parser.add_argument('--mongodb_connection', type=str, required=True, help=\"mongodb连接\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_database', type=str, required=True, help=\"mongodb数据库名\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_read_table', type=str, required=True, help=\"mongodb数据库读取表\")# tmp_one"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "id": "acc6aa2a-0f1d-43f3-985d-cd37c4a5ac6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Output:\n",
-      "Program starts execution!\n",
-      "args: Namespace(numerical_features=['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'], categorical_features=[], label='实际功率', learning_rate=0.01, num_leaves=31, min_data_in_leaf=21, mongodb_connection='mongodb://root:Jydl3377@192.168.1.33:27017', mongodb_database='algorithm_platform', mongodb_read_table='J00260NWP_records', mongodb_write_table='models', model_name='lightgbm_model')\n",
-      "df:************                         _id               C_TIME  ...   NWP-湿度    NWP-气压\n",
-      "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  ...  23.4918  892.3327\n",
-      "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  ...  24.0436  892.3809\n",
-      "\n",
-      "[2 rows x 34 columns]\n",
-      "Starting training...\n",
-      "[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000981 seconds.\n",
-      "You can set `force_col_wise=true` to remove the overhead.\n",
-      "[LightGBM] [Info] Total Bins 6885\n",
-      "[LightGBM] [Info] Number of data points in the train set: 7770, number of used features: 27\n",
-      "[LightGBM] [Info] Start training from score 47.616663\n",
-      "Training completed!\n",
-      "Collection 'models already exist, deleted successfully!\n",
-      "model inserted successfully!\n",
-      "Program execution ends!\n",
-      "\n",
-      "Error:\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 构建命令\n",
-    "command = [\n",
-    "    \"python\", \"lightgbm_model.py\",\n",
-    "    \"--numerical_features\", ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', \n",
-    "                             'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', \n",
-    "                             'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', \n",
-    "                             'NWP-温度', 'NWP-湿度', 'NWP-气压']),\n",
-    "    \"--categorical_features\", ','.join([]),\n",
-    "    \"--label\", \"实际功率\",\n",
-    "    \"--learning_rate\", \"0.01\",\n",
-    "    \"--num_leaves\", \"31\",\n",
-    "    \"--min_data_in_leaf\", \"21\",\n",
-    "    \"--mongodb_connection\", \"mongodb://root:Jydl3377@192.168.1.33:27017\",\n",
-    "    \"--mongodb_database\", \"algorithm_platform\",\n",
-    "    \"--mongodb_read_table\", \"J00260NWP_records\",\n",
-    "    \"--mongodb_write_table\", \"models\",\n",
-    "    \"--model_name\",\"lightgbm_model\"\n",
-    "]\n",
-    "\n",
-    "# 使用 subprocess.run 来执行命令\n",
-    "result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
-    "\n",
-    "# 输出结果和错误信息\n",
-    "print(\"Output:\")\n",
-    "print(result.stdout)\n",
-    "print(\"Error:\")\n",
-    "print(result.stderr)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "14a9feb4-5625-45d7-8e0f-6a8dea8c9f24",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run processing_limit_power_by_records   --timeBegin \"2024-05-01\"     --timeEnd \"2024-09-20\"   --station_code \"J00260\"     --col_time \"C_TIME\"    --mongondb_connection \"mongodb://root:Jydl3377@192.168.1.33:27017\"     --mongondb_database \"algorithm_platform\"     --mongondb_read_table \"J00260NWP_machines\"     --mongondb_write_table \"J00260NWP_records\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c0d791f8-12d8-4ab9-b561-b474e2a739f0",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python (panddlets)",
-   "language": "python",
-   "name": "panddlets"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.19"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

models_processing/model_training_lighgbm/Dockerfile

@@ -1,16 +0,0 @@
-# 使用官方 Python 镜像作为基础镜像
-FROM 192.168.1.36:5000/python:3.9
-ENV LANG=en_US.UTF-8
-# 设置工作目录
-WORKDIR /app
-
-# 复制当前目录内容到工作目录
-COPY . /app
-
-# 安装依赖项（如果有）
-RUN set -ex \
-    && pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-# 设置 ENTRYPOINT 为 Python 脚本
-ENTRYPOINT ["python", "model_training_lightgbm.py"]
-# 执行 Python 脚本
-CMD []

models_processing/model_training_lighgbm/requirements.txt

@@ -1,7 +0,0 @@
-pymongo==4.9.1
-pandas==2.1.4
-numpy==1.26.4
-lightgbm==4.3.0
-scikit-learn==1.4.2
-Flask==3.0.3
-waitress==3.0.0

models_processing/model_training_lighgbm/test.ipynb

@@ -1,334 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "660798fa-cf08-4cfd-b553-c6d750d7e7ad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import algo_test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "ff42670b-301d-4ee1-8da2-6a658fc5f286",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'_id': ObjectId('66ebba82a3b7dbe82538e34a'), 'name': 'Alice', 'age': 25, 'city': 'Beijing'}\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 使用 subprocess 执行 .py 文件\n",
-    "result = subprocess.run(['python', 'algo_test.py'], capture_output=True, text=True)\n",
-    "\n",
-    "# 打印输出\n",
-    "print(result.stdout)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "b47d615a-5953-43c0-9289-387d5c823a7a",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-09-26 16:09:24,040 - waitress - INFO - Serving on http://0.0.0.0:10090\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-09-26 16:11:05,381 - model_training_lightgbm log - INFO - {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度, NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度, NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n",
-      "features:************ ['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', ' NWP-气压']\n",
-      "Program execution ends!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-09-26 16:11:46,517 - model_training_lightgbm log - INFO - {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n",
-      "features:************ ['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压']\n",
-      "Starting training...\n",
-      "Training completed!\n",
-      "Collection 'models already exist, deleted successfully!\n",
-      "model inserted successfully!\n",
-      "Program execution ends!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-09-26 16:11:57,030 - model_training_lightgbm log - INFO - {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ' '}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ' '}\n",
-      "features:************ ['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压', ' ']\n",
-      "Program execution ends!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-09-26 16:12:20,633 - model_training_lightgbm log - INFO - {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': \"''\"}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'numerical_features': 'NWP预测总辐射,NWP-10米预测风速,NWP-10米预测风向,NWP-30米预测风速,NWP-30米预测风向,NWP-50米预测风速,NWP-50米预测风向,NWP-70米预测风速,NWP-70米预测风向,NWP-90米预测风速,NWP-90米预测风向,NWP-110米预测风速,NWP-110米预测风向,NWP-150米预测风速,NWP-150米预测风向,NWP-170米预测风速,NWP-170米预测风向,NWP-低云量,NWP-中云量,NWP-高云量,NWP-总云量,NWP-晴空总辐射,NWP-法相直辐射,NWP-总降水强度,NWP-温度,NWP-湿度,NWP-气压', 'label': '实际功率', 'learning_rate': '0.01', 'num_leaves': '31', 'min_data_in_leaf': '21', 'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'col_pp': 'predict', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': \"''\"}\n",
-      "features:************ ['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压', \"''\"]\n",
-      "Program execution ends!\n",
-      "server start!\n"
-     ]
-    }
-   ],
-   "source": [
-    " %run model_training_lightgbm.py"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "752c3ab3-cdef-47b0-9d9a-805348d30d6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "3.9.19 (main, May  6 2024, 20:12:36) [MSC v.1916 64 bit (AMD64)]\n"
-     ]
-    }
-   ],
-   "source": [
-    "parser.add_argument('--numerical_features', type=str, required=True, help=\"数值型特征\") \n",
-    "parser.add_argument('--categorical_features', type=str, required=True, help=\"类别型特征\")\n",
-    "parser.add_argument('--learning_rate', type=float, required=True, help=\"学习率\")\n",
-    "parser.add_argument('--num_leaves', type=int, required=True, help=\"叶子节点数\")\n",
-    "parser.add_argument('--min_data_in_leaf', type=int, required=True, help=\"叶子节点最小数据量\")# mongodb://root:Jydl3377@192.168.1.33:27017\n",
-    "parser.add_argument('--mongodb_connection', type=str, required=True, help=\"mongodb连接\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_database', type=str, required=True, help=\"mongodb数据库名\")# algorithm_platform\n",
-    "parser.add_argument('--mongodb_read_table', type=str, required=True, help=\"mongodb数据库读取表\")# tmp_one"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "id": "acc6aa2a-0f1d-43f3-985d-cd37c4a5ac6e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Output:\n",
-      "Program starts execution!\n",
-      "args: Namespace(numerical_features=['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向', 'NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向', 'NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向', 'NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向', 'NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射', 'NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'], categorical_features=[], label='实际功率', learning_rate=0.01, num_leaves=31, min_data_in_leaf=21, mongodb_connection='mongodb://root:Jydl3377@192.168.1.33:27017', mongodb_database='algorithm_platform', mongodb_read_table='J00260NWP_records', mongodb_write_table='models', model_name='lightgbm_model')\n",
-      "df:************                         _id               C_TIME  ...   NWP-湿度    NWP-气压\n",
-      "0  66ecc86f1075358d129a1d56  2024-05-02 22:30:00  ...  23.4918  892.3327\n",
-      "1  66ecc86f1075358d129a1d57  2024-05-02 22:45:00  ...  24.0436  892.3809\n",
-      "\n",
-      "[2 rows x 34 columns]\n",
-      "Starting training...\n",
-      "[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000981 seconds.\n",
-      "You can set `force_col_wise=true` to remove the overhead.\n",
-      "[LightGBM] [Info] Total Bins 6885\n",
-      "[LightGBM] [Info] Number of data points in the train set: 7770, number of used features: 27\n",
-      "[LightGBM] [Info] Start training from score 47.616663\n",
-      "Training completed!\n",
-      "Collection 'models already exist, deleted successfully!\n",
-      "model inserted successfully!\n",
-      "Program execution ends!\n",
-      "\n",
-      "Error:\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "import subprocess\n",
-    "\n",
-    "# 构建命令\n",
-    "command = [\n",
-    "    \"python\", \"lightgbm_model.py\",\n",
-    "    \"--numerical_features\", ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', \n",
-    "                             'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', \n",
-    "                             'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', \n",
-    "                             'NWP-温度', 'NWP-湿度', 'NWP-气压']),\n",
-    "    \"--categorical_features\", ','.join([]),\n",
-    "    \"--label\", \"实际功率\",\n",
-    "    \"--learning_rate\", \"0.01\",\n",
-    "    \"--num_leaves\", \"31\",\n",
-    "    \"--min_data_in_leaf\", \"21\",\n",
-    "    \"--mongodb_connection\", \"mongodb://root:Jydl3377@192.168.1.33:27017\",\n",
-    "    \"--mongodb_database\", \"algorithm_platform\",\n",
-    "    \"--mongodb_read_table\", \"J00260NWP_records\",\n",
-    "    \"--mongodb_write_table\", \"models\",\n",
-    "    \"--model_name\",\"lightgbm_model\"\n",
-    "]\n",
-    "\n",
-    "# 使用 subprocess.run 来执行命令\n",
-    "result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
-    "\n",
-    "# 输出结果和错误信息\n",
-    "print(\"Output:\")\n",
-    "print(result.stdout)\n",
-    "print(\"Error:\")\n",
-    "print(result.stderr)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "14a9feb4-5625-45d7-8e0f-6a8dea8c9f24",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run lightgbm_model.py   --numerical_features \"2024-05-01\"     --categorical_features \"2024-09-20\"   --label \"J00260\"     --learning_rate \"C_TIME\"    --num_leaves \"mongodb://root:Jydl3377@192.168.1.33:27017\"     --mongondb_database \"algorithm_platform\"     --mongondb_read_table \"J00260NWP_machines\"     --mongondb_write_table \"J00260NWP_records\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c0d791f8-12d8-4ab9-b561-b474e2a739f0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "docker run lightgbm_model.py --numerical_features ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压'])  --categorical_features  ','.join([])  --label  \"实际功率\" --learning_rate  \"0.01\" --num_leaves \"31\"  --min_data_in_leaf  \"21\"  --mongodb_connection  \"mongodb://root:Jydl3377@192.168.1.33:27017\" --mongodb_database  \"algorithm_platform\" --mongodb_read_table  \"J00260NWP_records\" --mongodb_write_table  \"models\" --model_name \"lightgbm_model\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8bdd9bd2-4373-4465-b20a-de574fb29436",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-11-21 14:54:47,009 - waitress - INFO - Serving on http://0.0.0.0:10089\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-11-21 14:54:51,021 - model_training_lightgbm log - INFO - {'numerical_features': 'temperature2,direction10,speed10,mcc,solarZenith,dniCalcd,humidity2,hcc,clearskyGhi,lcc,tcc,surfacePressure,tpr,radiation', 'label': 'realPower', 'model_params': \"{'num_leaves': 31,         'learning_rate': 0.1,         'min_data_in_leaf': 21}\", 'num_boost_round': '500', 'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Program starts execution!\n",
-      "args {'numerical_features': 'temperature2,direction10,speed10,mcc,solarZenith,dniCalcd,humidity2,hcc,clearskyGhi,lcc,tcc,surfacePressure,tpr,radiation', 'label': 'realPower', 'model_params': \"{'num_leaves': 31,         'learning_rate': 0.1,         'min_data_in_leaf': 21}\", 'num_boost_round': '500', 'mongodb_database': 'db2', 'mongodb_read_table': 'j00629_copy_statistics', 'mongodb_write_table': 'models', 'model_name': 'lightgbm_model', 'categorical_features': ''}\n",
-      "features:************ ['temperature2', 'direction10', 'speed10', 'mcc', 'solarZenith', 'dniCalcd', 'humidity2', 'hcc', 'clearskyGhi', 'lcc', 'tcc', 'surfacePressure', 'tpr', 'radiation']\n",
-      "Starting training...\n",
-      "The test rmse is: 7.095613313100767,\"The test mae is:\"3.0062923664706886\n",
-      "Training completed!\n",
-      "Collection 'models already exist, deleted successfully!\n",
-      "model inserted successfully!\n",
-      "Program execution ends!\n"
-     ]
-    }
-   ],
-   "source": [
-    " %run model_training_lightgbm.py"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "5ae69d4d-fa6e-4a26-95a4-232b55e469b8",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'a': 1, 'b': 3, 'c': 4}\n"
-     ]
-    }
-   ],
-   "source": [
-    "temperature2,direction10,speed10,mcc,solarZenith,dniCalcd,humidity2,hcc,clearskyGhi,lcc,tcc,surfacePressure,tpr,radiation"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python (panddlets)",
-   "language": "python",
-   "name": "panddlets"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.19"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

models_pytorch/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

models_pytorch/model_transformer.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :model_transformer.py
+# @Time      :2024/12/4 08:43
+# @Author    :David
+# @Company: shenyang JY
+
+
+if __name__ == "__main__":
+    run_code = 0

models_sklearn/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

models_sklearn/model_forest_cdq.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2024/10/23 13:04
+# file: forest.py
+# author: David
+# company: shenyang JY
+from pathlib import Path
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.ensemble import RandomForestRegressor
+import joblib
+import threading
+
+
+class Forest(object):
+    lock = threading.Lock()
+    gbdt1 = None
+    gbdt2 = None
+    gbdt3 = None
+    gbdt4 = None
+
+    def __init__(self, log, args):
+        self.args = args
+        self.logger = log
+        self.opt = self.args.parse_args_and_yaml()
+        self.hs = []
+        for fea in ['C_REAL_VALUE'] + [self.opt.usable_power["env"]]:
+            self.hs += [fea + str(i + 1) for i in range(self.opt.history_env_hours * 4)]
+        self.set_errors()
+
+    def training_model(self, opt):
+        cols = opt.data['columns'] + self.hs + ['C_FP_VALUE']
+        data_path = Path('./cache/data/')
+        files = list(data_path.rglob("lgb_data*.csv"))
+        for file in files:
+            data = pd.read_csv(file)
+            df_train_x, df_train_y = data[cols].values, data['error'].values
+            # train_x, val_x, train_y, val_y = train_test_split(df_train_x, df_train_y, test_size=0.1, shuffle=False)
+            # 敲定好一组参数
+            params_grid = {
+                'n_estimators': [50, 100, 200, 500],
+                'max_features': [None, 'sqrt', 'log2'],
+                'max_depth': [4, 6, 8, 10],
+                # 'criterion': ['squared_error', 'absolute_error']
+            }
+
+            rf = RandomForestRegressor(random_state=42)
+
+            grid_search = GridSearchCV(estimator=rf, param_grid=params_grid, cv=5, n_jobs=None, verbose=2, scoring='neg_mean_squared_error')
+
+            grid_search.fit(df_train_x, df_train_y)
+
+            # 输出最佳参数和最佳得分
+            self.logger.info(f"Best parameters found: {grid_search.best_params_}")
+            print(f"Best cross-validation score: {-grid_search.best_score_}")
+
+            # 使用最佳模型进行预测
+            best_model = grid_search.best_estimator_
+            # pred_y = best_model.predict(val_x)
+
+            # 计算测试集上的均方误差
+            # mse = mean_squared_error(val_y, pred_y)
+            # print(f"Mean Squared Error on test set: {mse}")
+
+            # 保存模型
+            self.logger.info('保存模型...')
+            joblib.dump(best_model, './var/lgb_model_{}.pkl'.format(str(file)[-5]))
+
+    @classmethod
+    def set_errors(cls):
+        try:
+            with cls.lock:
+                cls.gbdt1 = joblib.load('./var/lgb_model_1.pkl')
+                cls.gbdt2 = joblib.load('./var/lgb_model_2.pkl')
+                cls.gbdt3 = joblib.load('./var/lgb_model_3.pkl')
+                cls.gbdt4 = joblib.load('./var/lgb_model_4.pkl')
+        except Exception as e:
+            print("加载模型权重失败:{}".format(e.args))
+
+    def predict_error_clock(self, hour, data, api=False):
+        cols = self.opt.data['columns'] + self.hs + ['C_FP_VALUE']
+        if hour == 1:
+            gbdt = Forest.gbdt1
+        elif hour == 2:
+            gbdt = Forest.gbdt2
+            self.logger.info("预测模型，地址：{}".format(id(gbdt)))
+        elif hour == 3:
+            gbdt = Forest.gbdt3
+        else:
+            gbdt = Forest.gbdt4
+
+        if api:
+            dq = data['C_FP_VALUE']
+            eat_data = data[cols].values[np.newaxis, :]
+            error_predict = gbdt.predict(eat_data)[0]
+            dq_fix = round(dq + error_predict, 2)
+            dq_fix = dq_fix if dq_fix > 0 else 0
+            dq_fix = self.opt.cap if dq_fix > self.opt.cap else dq_fix
+        else:
+            dq = data['C_FP_VALUE'].values
+            eat_data = data[cols].values
+            error_predict = gbdt.predict(eat_data)
+            dq_fix = dq + error_predict
+            dq_fix[dq_fix < 0] = 0  # 如果出现负数，置为0
+            dq_fix[dq_fix > self.opt.cap] = self.opt.cap  # 出现大于实际装机量的数，置为实际装机量
+            dq_fix = np.around(dq_fix, decimals=2)
+
+        return dq_fix
+
+    def predict_error(self, pre_data):
+        dq_fixs, ctimes = [], []
+        for point, data in pre_data.iterrows():
+            if point < 4:
+                hour = 1
+            elif point < 8:
+                hour = 2
+            elif point < 12:
+                hour = 3
+            else:
+                hour = 4
+            dq_fix = self.predict_error_clock(hour, data, api=True)
+            dq_fixs.append(dq_fix)
+        return dq_fixs
+
+
+
+if __name__ == '__main__':
+    from config import myargparse
+    from logs import Log
+
+    args = myargparse(discription="场站端配置", add_help=False)
+    log = Log()

models_sklearn/model_lightgbm_cdq.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :model_lightgbm_cdq.py
+# @Time      :2024/12/4 08:55
+# @Author    :David
+# @Company: shenyang JY
+
+
+if __name__ == "__main__":
+    run_code = 0

models_tensorflow/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

models_tensorflow/losses.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/5/8 13:15
+# file: loss.py.py
+# author: David
+# company: shenyang JY
+from keras import backend as K
+import tensorflow as tf
+tf.compat.v1.set_random_seed(1234)
+
+
+class SouthLoss(tf.keras.losses.Loss):
+    def __init__(self, opt, name='south_loss'):
+        """
+        南网新规则损失函数
+        :param cap:装机容量
+        """
+        super(SouthLoss, self).__init__(name=name)
+        self.cap = opt.cap*0.2    # 没有归一化cap，必须要先进行归一化
+        self.opt = opt
+        # self.cap01 = opt.cap*0.1
+
+    def call(self, y_true, y_predict):
+        """
+        自动调用
+        :param y_true: 标签
+        :param y_predict: 预测
+        :return: 损失值
+        """
+        # 计算实际和预测的差值
+        y_true = y_true * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_predict = y_predict * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_true = y_true[:, 15]
+        y_predict = y_predict[:, 15]
+        diff = y_true - y_predict
+        logistic_values = tf.sigmoid(10000 * (y_true - self.cap))
+        base = logistic_values * y_true + (1-logistic_values)*self.cap
+        loss = K.square(diff/base)
+        # loss = K.mean(loss, axis=-1)
+        return loss
+
+    def call2(self, y_true, y_predict):
+        y_true = y_true * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_predict = y_predict * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_true = y_true[:, 15]
+        y_predict = y_predict[:, 15]
+        diff = y_true - y_predict
+        logistic_values = tf.sigmoid(10000 * (y_true - self.cap))
+        base = logistic_values * y_true + (1 - logistic_values) * self.cap
+        loss = K.square(diff / base)
+
+        mask_logical = tf.logical_and(tf.greater(y_true, self.cap01), tf.greater(y_predict, self.cap01))
+        count = tf.reduce_sum(tf.cast(mask_logical, tf.float32), axis=-1)
+        safe_count = tf.maximum(count, 1)
+        # reduce_sum_loss = tf.reduce_sum(loss, axis=-1)
+        mean_loss = loss / safe_count
+        return mean_loss
+
+    def call1(self, y_true, y_predict):
+        y_true = y_true * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_predict = y_predict * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        base = tf.where(y_true > self.cap, y_true, tf.ones_like(y_true)*self.cap)
+        loss = (y_true - y_predict) / base
+        squared_loss = tf.square(loss)
+        mean_squared_loss = tf.reduce_mean(squared_loss, axis=[1])
+        return  mean_squared_loss
+
+
+class NorthEastLoss(tf.keras.losses.Loss):
+    def __init__(self, opt, name='northeast_loss'):
+        """
+        东北新规则超短期损失函数
+        """
+        super(NorthEastLoss, self).__init__(name=name)
+        self.opt = opt
+        self.cap = round(opt.cap*0.1, 2)
+
+    def call(self, y_true, y_predict):
+        # 这里我们添加了一个小的 epsilon 值来避免除以 0
+        y_true = y_true * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+        y_predict = y_predict * self.opt.std['C_REAL_VALUE'] + self.opt.mean['C_REAL_VALUE']
+
+        mask_logical = tf.logical_and(tf.greater(y_true, self.cap), tf.greater(y_predict, self.cap))
+        # mask = tf.cast(~mask_logical, tf.float32)
+        # y_true = y_true * (1 - mask) + 0 * mask
+        # y_predict = y_predict * (1 - mask) + 0 * mask
+
+
+        epsilon = tf.keras.backend.epsilon()
+        y_predict_safe = y_predict + epsilon
+
+        # 计算 (y_true - y_predict) / y_predict_safe
+        difference_over_predict = tf.abs(y_predict - y_true) / tf.abs(y_predict_safe)
+
+        # 将结果中大于等于 1 的部分置为 1，剩下的保留原值
+        masked_difference = tf.where(difference_over_predict >= 1, tf.ones_like(difference_over_predict)*1, difference_over_predict) #tf.where的操作是逐元素的，并且它不会改变张量中元素的数学性质（如可微性、可导性）。
+
+        # 这里我们先沿着特征维度求和，但你也可以选择平均（使用 tf.reduce_mean 而不是 tf.reduce_sum）
+        count = tf.reduce_sum(tf.cast(mask_logical, tf.float32), axis=-1)
+        sum_diff = tf.reduce_sum(masked_difference, axis=-1)
+        # mean_loss = tf.reduce_mean(masked_difference, axis=[1])
+        safe_count = tf.maximum(count, 1)
+        mean = sum_diff / safe_count
+        mean1 = tf.reduce_sum(masked_difference, axis=-1)
+        return mean
+
+
+class NorthWestLoss(tf.keras.losses.Loss):
+    def __init__(self, name='northwest_loss'):
+        """
+        东北新规则超短期损失函数
+        """
+        super(NorthWestLoss, self).__init__(name=name)
+
+    def call(self, y_true, y_pred):
+        # 保证预测值和真实值是浮点数
+        y_pred = tf.cast(y_pred, tf.float32)
+        y_true = tf.cast(y_true, tf.float32)
+
+        # 避免除零错误
+        epsilon = 1e-8
+        y_pred_adjusted = y_pred + epsilon
+        y_true_adjusted = y_true + epsilon
+
+        # 计算 |Pr - Pn|
+        abs_diff = tf.abs(y_pred - y_true)
+
+        # 计算 |Pr - Pn| 的总和
+        sum_abs_diff = tf.reduce_sum(abs_diff)
+
+        # 计算每个差值的权重 |Pr - Pn| / sum(|Pr - Pn|)
+        weights = abs_diff / (sum_abs_diff + epsilon)  # 添加 epsilon 避免除零
+
+        # 计算 |Pr/(Pr + Pn) - 0.5|
+        ratios = tf.abs((y_pred_adjusted / (y_pred_adjusted + y_true_adjusted)) - 0.5)
+
+        # 计算最终的损失值
+        loss = 1.0 - 2.0 * tf.reduce_sum(ratios * weights)
+        return loss

models_tensorflow/model_bp_cdq.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2024/5/6 13:25
+# file: time_series.py
+# author: David
+# company: shenyang JY
+import os.path
+
+from flask import Flask
+from keras.layers import Input, Dense, LSTM, concatenate, Conv1D, Conv2D, MaxPooling1D, BatchNormalization, Flatten, Dropout, Reshape, Lambda, TimeDistributed
+from keras.models import Model, load_model
+from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
+from keras import optimizers, regularizers
+import keras.backend as K
+import numpy as np
+np.random.seed(42)
+from sloss import SouthLoss, NorthEastLoss
+import tensorflow as tf
+tf.compat.v1.set_random_seed(1234)
+from threading import Lock
+model_lock = Lock()
+
+
+def rmse(y_true, y_pred):
+    return K.sqrt(K.mean(K.square(y_pred - y_true)))
+
+
+def mae(y_true, y_pred):
+    return K.mean(K.abs(y_pred - y_true), axis=-1)
+
+
+var_dir = os.path.dirname(os.path.dirname(__file__))
+
+
+class FMI(object):
+    model = None
+    train = False
+
+    def __init__(self, log, args, graph, sess):
+        self.logger = log
+        self.graph = graph
+        self.sess = sess
+        opt = args.parse_args_and_yaml()
+        with self.graph.as_default():
+            tf.compat.v1.keras.backend.set_session(self.sess)
+            FMI.get_model(opt)
+
+    @staticmethod
+    def get_model(opt):
+        """
+        单例模式+线程锁，防止在异步加载时引发线程安全
+        """
+        try:
+            if FMI.model is None or FMI.train is True:
+                with model_lock:
+                    FMI.model = FMI.get_keras_model(opt)
+                    FMI.model.load_weights(os.path.join(var_dir, 'var', 'fmi.h5'))
+        except Exception as e:
+            print("加载模型权重失败:{}".format(e.args))
+
+    @staticmethod
+    def get_keras_model(opt):
+        db_loss = NorthEastLoss(opt)
+        # south_loss = SouthLoss(opt)
+        l1_reg = regularizers.l1(opt.Model['lambda_value_1'])
+        l2_reg = regularizers.l2(opt.Model['lambda_value_2'])
+        nwp_input = Input(shape=(opt.Model['time_step'], opt.Model['input_size_nwp']), name='nwp')
+        env_input = Input(shape=(opt.Model['his_points'], opt.Model['input_size_env']), name='env')
+
+        con1 = Conv1D(filters=64, kernel_size=1, strides=1, padding='valid', activation='relu',
+                      kernel_regularizer=l2_reg)(nwp_input)
+        d1 = Dense(32, activation='relu', name='d1', kernel_regularizer=l1_reg)(con1)
+        nwp = Dense(8, activation='relu', name='d2', kernel_regularizer=l1_reg)(d1)
+
+        con2 = Conv1D(filters=64, kernel_size=5, strides=1, padding='valid', activation='relu', kernel_regularizer=l2_reg)(env_input)
+        env = MaxPooling1D(pool_size=5, strides=1, padding='valid', data_format='channels_last')(con2)
+        for i in range(opt.Model['lstm_layers']):
+            rs = True
+            if i == opt.Model['lstm_layers']-1:
+                rs = False
+            env = LSTM(units=opt.Model['hidden_size'], return_sequences=rs, name='env_lstm'+str(i), kernel_regularizer=l2_reg)(env)
+        tiao = Dense(16, name='d4', kernel_regularizer=l1_reg)(env)
+
+        if opt.Model['fusion']:
+            nwpf = Flatten()(nwp)
+            fusion = concatenate([nwpf, tiao])
+        else:
+            fusion = Flatten()(nwp)
+
+        output = Dense(opt.Model['output_size'], name='d5')(fusion)
+        model = Model([env_input, nwp_input], output)
+        adam = optimizers.Adam(learning_rate=opt.Model['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=1e-7,
+                               amsgrad=True)
+        model.compile(loss=rmse, optimizer=adam)
+        return model
+
+    def train_init(self, opt):
+        tf.compat.v1.keras.backend.set_session(self.sess)
+        model = FMI.get_keras_model(opt)
+        try:
+            if opt.Model['add_train'] and opt.authentication['repair'] != "null":
+                # 进行加强训练，支持修模
+                model.load_weights(os.path.join(var_dir, 'var', 'fmi.h5'))
+                self.logger.info("已加载加强训练基础模型")
+        except Exception as e:
+            self.logger.info("加强训练加载模型权重失败:{}".format(e.args))
+        model.summary()
+        return model
+
+    def training(self, opt, train_and_valid_data):
+        model = self.train_init(opt)
+        train_X, train_Y, valid_X, valid_Y = train_and_valid_data
+        print("----------", np.array(train_X[0]).shape)
+        print("++++++++++", np.array(train_X[1]).shape)
+        # weight_lstm_1, bias_lstm_1 = model.get_layer('d1').get_weights()
+        # print("weight_lstm_1 = ", weight_lstm_1)
+        # print("bias_lstm_1 = ", bias_lstm_1)
+
+        check_point = ModelCheckpoint(filepath='./var/' + 'fmi.h5', monitor='val_loss',
+                                      save_best_only=True, mode='auto')
+        early_stop = EarlyStopping(monitor='val_loss', patience=opt.Model['patience'], mode='auto')
+        # tbCallBack = TensorBoard(log_dir='../figure',
+        #                          histogram_freq=0,
+        #                          write_graph=True,
+        #                          write_images=True)
+        history = model.fit(train_X, train_Y, batch_size=opt.Model['batch_size'], epochs=opt.Model['epoch'], verbose=2,
+                            validation_data=(valid_X, valid_Y), callbacks=[check_point, early_stop])
+        loss = np.round(history.history['loss'], decimals=5)
+        val_loss = np.round(history.history['val_loss'], decimals=5)
+        self.logger.info("-----模型训练经过{}轮迭代-----".format(len(loss)))
+        self.logger.info("训练集损失函数为：{}".format(loss))
+        self.logger.info("验证集损失函数为：{}".format(val_loss))
+        self.logger.info("训练结束，原模型地址：{}".format(id(FMI.model)))
+        with self.graph.as_default():
+            tf.compat.v1.keras.backend.set_session(self.sess)
+            FMI.train = True
+            FMI.get_model(opt)
+            FMI.train = False
+        self.logger.info("保护线程，加载模型，地址：{}".format(id(FMI.model)))
+
+    def predict(self, test_X, batch_size=1):
+        with self.graph.as_default():
+            with self.sess.as_default():
+                result = FMI.model.predict(test_X, batch_size=batch_size)
+        self.logger.info("执行预测方法")
+        return result

models_tensorflow/model_cnn_lstm_cdq.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2024/5/6 13:25
+# file: time_series.py
+# author: David
+# company: shenyang JY
+import os.path
+from keras.layers import Input, Dense, LSTM, concatenate, Conv1D, Conv2D, MaxPooling1D, BatchNormalization, Flatten, Dropout
+from keras.models import Model, load_model
+from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
+from keras import optimizers, regularizers
+import keras.backend as K
+import numpy as np
+from sqlalchemy.ext.instrumentation import find_native_user_instrumentation_hook
+
+np.random.seed(42)
+from cache.sloss import SouthLoss, NorthEastLoss
+import tensorflow as tf
+tf.compat.v1.set_random_seed(1234)
+from threading import Lock
+model_lock = Lock()
+
+def rmse(y_true, y_pred):
+    return K.sqrt(K.mean(K.square(y_pred - y_true)))
+
+
+var_dir = os.path.dirname(os.path.dirname(__file__))
+
+
+class FMI(object):
+    model = None
+    train = False
+
+    def __init__(self, log, args, graph, sess):
+        self.logger = log
+        self.graph = graph
+        self.sess = sess
+        opt = args.parse_args_and_yaml()
+        with self.graph.as_default():
+            tf.compat.v1.keras.backend.set_session(self.sess)
+            FMI.get_model(opt)
+
+    @staticmethod
+    def get_model(opt):
+        """
+        单例模式+线程锁，防止在异步加载时引发线程安全
+        """
+        try:
+            if FMI.model is None or FMI.train is True:
+                with model_lock:
+                    FMI.model = FMI.get_keras_model(opt)
+                    FMI.model.load_weights(os.path.join(var_dir, 'var', 'fmi.h5'))
+        except Exception as e:
+            print("加载模型权重失败:{}".format(e.args))
+
+    @staticmethod
+    def get_keras_model(opt):
+        db_loss = NorthEastLoss(opt)
+        south_loss = SouthLoss(opt)
+        l1_reg = regularizers.l1(opt.Model['lambda_value_1'])
+        l2_reg = regularizers.l2(opt.Model['lambda_value_2'])
+        nwp_input = Input(shape=(opt.Model['time_step'], opt.Model['input_size_nwp']), name='nwp')
+        env_input = Input(shape=(opt.Model['his_points'], opt.Model['input_size_env']), name='env')
+
+        con1 = Conv1D(filters=64, kernel_size=5, strides=1, padding='valid', activation='relu', kernel_regularizer=l2_reg)(nwp_input)
+        nwp = MaxPooling1D(pool_size=5, strides=1, padding='valid', data_format='channels_last')(con1)
+        nwp_lstm = LSTM(units=opt.Model['hidden_size'], return_sequences=False, kernel_regularizer=l2_reg)(nwp)
+
+        con2 = Conv1D(filters=64, kernel_size=5, strides=1, padding='valid', activation='relu', kernel_regularizer=l2_reg)(env_input)
+        env = MaxPooling1D(pool_size=5, strides=1, padding='valid', data_format='channels_last')(con2)
+        env_lstm = LSTM(units=opt.Model['hidden_size'], return_sequences=False, name='env_lstm',kernel_regularizer=l2_reg)(env)
+        tiao = Dense(4, name='d4', kernel_regularizer=l1_reg)(env_lstm)
+
+        if opt.Model['fusion']:
+            fusion = concatenate([nwp_lstm, tiao])
+        else:
+            fusion = nwp_lstm
+
+        output = Dense(opt.Model['output_size'], name='cdq_output')(fusion)
+
+        model = Model([env_input, nwp_input], output)
+        adam = optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, amsgrad=True)
+        model.compile(loss=south_loss, optimizer=adam)
+        return model
+
+    def train_init(self, opt):
+        tf.compat.v1.keras.backend.set_session(self.sess)
+        model = FMI.get_keras_model(opt)
+        try:
+            if opt.Model['add_train'] and opt.authentication['repair'] != "null":
+                # 进行加强训练，支持修模
+                model.load_weights(os.path.join(var_dir, 'var', 'fmi.h5'))
+                self.logger.info("已加载加强训练基础模型")
+        except Exception as e:
+            self.logger.info("加强训练加载模型权重失败:{}".format(e.args))
+        model.summary()
+        return model
+
+    def training(self, opt, train_and_valid_data):
+        model = self.train_init(opt)
+        train_X, train_Y, valid_X, valid_Y = train_and_valid_data
+        print("----------", np.array(train_X[0]).shape)
+        print("++++++++++", np.array(train_X[1]).shape)
+        # weight_lstm_1, bias_lstm_1 = model.get_layer('d1').get_weights()
+        # print("weight_lstm_1 = ", weight_lstm_1)
+        # print("bias_lstm_1 = ", bias_lstm_1)
+
+        check_point = ModelCheckpoint(filepath='./var/' + 'fmi.h5', monitor='val_loss',
+                                      save_best_only=True, mode='auto')
+        early_stop = EarlyStopping(monitor='val_loss', patience=opt.Model['patience'], mode='auto')
+        # tbCallBack = TensorBoard(log_dir='../figure',
+        #                          histogram_freq=0,
+        #                          write_graph=True,
+        #                          write_images=True)
+        history = model.fit(train_X, train_Y, batch_size=opt.Model['batch_size'], epochs=opt.Model['epoch'], verbose=2,
+                            validation_data=(valid_X, valid_Y), callbacks=[check_point, early_stop])
+        loss = np.round(history.history['loss'], decimals=5)
+        val_loss = np.round(history.history['val_loss'], decimals=5)
+        self.logger.info("-----模型训练经过{}轮迭代-----".format(len(loss)))
+        self.logger.info("训练集损失函数为：{}".format(loss))
+        self.logger.info("验证集损失函数为：{}".format(val_loss))
+        self.logger.info("训练结束，原模型地址：{}".format(id(FMI.model)))
+        with self.graph.as_default():
+            tf.compat.v1.keras.backend.set_session(self.sess)
+            FMI.train = True
+            FMI.get_model(opt)
+            FMI.train = False
+        self.logger.info("保护线程，加载模型，地址：{}".format(id(FMI.model)))
+
+    def predict(self, test_X, batch_size=1):
+        with self.graph.as_default():
+            with self.sess.as_default():
+                result = FMI.model.predict(test_X, batch_size=batch_size)
+        self.logger.info("执行预测方法")
+        return result
+
+    def train_custom(self, train_X, train_Y, model, opt):
+        epochs = opt.Model['epoch']
+        batch_size = opt.Model['batch_size']
+        num_batches = len(train_X) // batch_size    # 取整
+
+        optimizer = tf.keras.optimizers.Adam(learning_rate=opt.Model[""])
+        for epoch in range(epochs):
+            for batch_index in range(epochs):
+                start = batch_index * batch_size
+                end = start + batch_size
+                x_batch, y_batch = train_X[start: end], train_Y[start: end]
+
+                with tf.GradientTape() as tape:
+                    res = model(x_batch)
+                    loss = rmse(y_batch, res)
+
+                gradients = tape.gradient(loss, model.trainable_variables)
+
+

models_tensorflow/model_multi_task_cdq.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :model_weather_fix_cdq.py
+# @Time      :2024/12/4 08:50
+# @Author    :David
+# @Company: shenyang JY
+
+
+if __name__ == "__main__":
+    run_code = 0

post_evaluation/formula.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :formula.py
+# @Time      :2024/12/4 08:47
+# @Author    :David
+# @Company: shenyang JY
+
+import numpy as np
+import pandas as pd
+'''
+准确率formula.py模块使用手顺：
+① Assessment: 考核类 Formular: 准确率计算类
+② 初始化并传参：初始化考核类，传递opt命名空间 (包括cap装机容量)、logger日志对象（也可以用print代替）
+③ 调用方法：按照考核类标注的方法和参数含义，进行传参和调用，获得日期平均准确率和考核分数
+----------
+注意：
+regulations > solar/wind []列表含义（按顺序）：短期准确率考核线，超短期准确率考核线，短期考核系数，超短期考核系数
+西北，东北地区短期采用考核积分电量和偏差带进行考核，这里不进行计算
+'''
+class Assessment(object):
+    def __init__(self, opt, logger):
+        self.logger = logger
+        self.formula = Formulas(opt)
+        self.regulations = {
+            "south": {
+                'func': self.formula.calculate_acc_south,
+                'solar': [65, 70, 1*0.2, 1*0.2],
+                'wind': [60, 65, 0.2*2, 0.2*2],
+                'check': True
+            },
+            "east":{
+                'func': self.formula.calculate_acc,
+                'solar': ['', 97, '', 0.09*1],
+                'wind': ['', 96, '', 0.09*1],
+                'check': False
+            },
+            "northeast":{
+                'func': self.formula.calculate_acc_northeast,
+                'solar': ['', 85, '', 0.02*0.1],
+                'wind': ['', 80, '', 0.02*0.1]
+            },
+            "northwest":{
+                'func': self.formula.calculate_acc_northwest,
+                'solar': ['', 75, '', 0.015*0.1*0.5],
+                'wind': ['', 75, '', 0.015*0.1*0.5]
+            }
+
+        }
+
+    def electricity_solar_cdq(self, df, province, predict, label='C_REAL_VALUE', output=True):
+        df['C_TIME'] = pd.to_datetime(df['C_TIME'])
+        df['C_TIME_DAY'] = df['C_TIME'].dt.strftime("%Y-%m-%d")
+        dfs = df.groupby('C_TIME_DAY')
+        limit = self.regulations[province]['solar'][1]
+        alpha = self.regulations[province]['solar'][3]
+        sum_score, sum_acc = 0, 0
+        for dt, data in dfs:
+            acc = self.regulations[province]['func'](data[predict].values, data[label].values)
+            score = (limit - acc) * alpha if acc < limit else 0
+            sum_acc += acc
+            sum_score += score
+            if output:
+                self.logger.info("预测名称：{}，日期：{}，区域：{}，超短期的准确率：{:.4f}，考核分数：{:.4f}".format(predict, str(dt), province, acc, score))
+        return round(sum_acc/len(dfs), 2), round(sum_score/len(dfs), 2)
+
+
+    def electricity_wind_cdq(self, df, province, predict, label='C_REAL_VALUE'):
+        df['C_TIME'] = pd.to_datetime(df['C_TIME'])
+        df['C_TIME_DAY'] = df['C_TIME'].dt.strftime("%Y-%m-%d")
+        dfs = df.groupby('C_TIME_DAY')
+        limit = self.regulations[province]['solar'][1]
+        alpha = self.regulations[province]['solar'][3]
+        sum_score, sum_acc = 0, 0
+        for dt, data in dfs:
+            acc = self.regulations[province]['func'](data[predict].values, data[label].values)
+            score = (limit - acc) * alpha  if acc < limit else 0
+            sum_acc += acc
+            sum_score += score
+            self.logger.info(
+                "日期：{}，区域：{}，超短期的准确率：{:.4f}，考核分数：{:.4f}".format(str(dt), province, acc, score))
+        return round(sum_acc / len(dfs), 2), round(sum_score / len(dfs), 2)
+
+
+class Formulas(object):
+    def __init__(self, opt):
+        self.opt = opt
+
+    def calculate_acc(self, label_data, predict_data):
+        loss = np.sum((label_data - predict_data) ** 2) / len(label_data)  # mse
+        loss_sqrt = np.sqrt(loss)  # rmse
+        loss_acc = (1 - loss_sqrt / self.opt.cap) * 100
+        return loss_acc
+
+
+    def calculate_acc_south(self, label_data, predict_data):
+        cap = 0.1 * self.opt.cap
+        mask = (label_data < cap) & (predict_data < cap)
+        label_data = label_data[~mask]
+        predict_data = predict_data[~mask]
+        diff = label_data - predict_data
+        base = np.where(label_data < self.opt.cap * 0.2, self.opt.cap * 0.2, label_data)
+        acc = np.sum((diff / base) ** 2) / len(diff)
+        acc = (1 - np.sqrt(acc)) * 100
+        return acc
+
+
+    def calculate_acc_northwest(self, label_data, predict_data):
+        cap = 0.03 * self.opt.cap
+        mask = (label_data < cap) & (predict_data < cap)
+        label_data = label_data[~mask]
+        predict_data = predict_data[~mask]
+        diff = np.abs(label_data - predict_data)
+        base1 = label_data + predict_data + 1e-9
+        base2 = np.sum(diff) + 1e-9
+        acc = (1 - 2 * np.sum(np.abs(label_data / base1 - 0.5) * diff / base2)) * 100
+        return acc
+
+    def calculate_acc_northeast(self, label_data, predict_data):
+        cap = 0.1 * self.opt.cap
+        mask = (label_data < cap) & (predict_data < cap)
+        label_data = label_data[~mask]
+        predict_data = predict_data[~mask]
+        diff = np.abs(predict_data - label_data)
+        deviation = diff / np.abs(predict_data + 1e-9)
+        acc = np.where(deviation >= 1, 1, deviation)
+        acc = 1 - np.mean(acc)
+        return acc
+
+if __name__ == '__main__':
+    from config import myargparse
+
+    args = myargparse(discription="场站端配置", add_help=False)
+    opt = args.parse_args_and_yaml()
+    formula = Formulas(opt)
+    test = pd.read_csv('./data/测试集16.csv')
+    test = test.iloc[:96, :]
+    acc = formula.calculate_acc_northeast(test['C_REAL_VALUE'], test['C_FP_VALUE'])
+    print(acc)

post_evaluation/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

post_processing/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

post_processing/spot_trading.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :spot_trading.py
+# @Time      :2024/12/3 19:29
+# @Author    :David
+# @Company: shenyang JY
+import datetime
+
+def cdq(self, fhs):
+    cdq = []
+    # sun_up_time = datetime.datetime.strptime(self.opt.first_point['sun_up_time'], '%Y-%m-%d %H:%M:%S')
+    for i, fh in fhs.iterrows():
+        if i + 1 > 16:
+            T = 'T16'
+        else:
+            T = 'T' + str(i + 1)
+        coe_m = float(self.opt.coe[T]['m'])
+        coe_n = float(self.opt.coe[T]['n'])
+        dt = fh['C_TIME']
+        dq = fh['C_FP_VALUE']
+        dq_fix = fh['dq_fix']
+        his_fix = fh['his_fix']
+        new_dq = round(coe_n * his_fix + coe_m * dq_fix, 2)  # 修改曲线为0问题
+
+        for i, cal in enumerate(self.opt.spot_trading):
+            if 'T' + str(cal['point']) == T:
+                new_dq = new_dq * cal['coe'] + cal['abs']
+                self.logger.info("现货交易：第{}个点，进行了乘系数加绝对值修改".format(cal['point']))
+
+        for i, cal in enumerate(self.opt.calculate):
+            interval = list(range(cal['extraMinRange'], cal['extraMaxRange'] + 1))
+            if dt.hour in interval:
+                new_dq = new_dq * cal['coe'] + cal['abs']
+                self.logger.info(
+                    "第{}组，时段为：{}-{}点，时刻：{}，进行了乘系数加绝对值修改".format(i + 1, cal['extraMinRange'],
+                                                                                    cal['extraMaxRange'],
+                                                                                    dt.strftime('%Y-%m-%d %H:%M:%S')))
+        new_dq = new_dq if dq > 0 else 0
+        new_dq = new_dq if new_dq > 0 else 0
+        new_dq = self.opt.cap if new_dq > self.opt.cap else new_dq
+        cdq.append({"C_TIME": dt.strftime('%Y-%m-%d %H:%M:%S'), "CDQ_VALUE": new_dq})
+    return cdq
+

pre_evaluation/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")

pre_evaluation/pearson.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :pearson.py
+# @Time      :2024/12/4 08:58
+# @Author    :David
+# @Company: shenyang JY
+
+
+if __name__ == "__main__":
+    run_code = 0

pre_evaluation/turbine_clustering.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :turbine_clustering.py
+# @Time      :2024/12/4 08:58
+# @Author    :David
+# @Company: shenyang JY
+
+
+if __name__ == "__main__":
+    run_code = 0

pre_processing/logs.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# time: 2023/3/4 22:28
+# file: myLog.py
+# author: David
+# company: shenyang JY
+
+"""
+1. 信息流以控制台和文件形式打印
+2. 文件保存以启动日期为文件名
+3. 控制台INFO，文件DEBUG
+"""
+import codecs
+from pathlib import Path
+import logging, logging.handlers, time, os, re
+from logging.handlers import BaseRotatingHandler
+
+
+class DailyRotatingFileHandler(BaseRotatingHandler):
+    """
+    同`logging.TimedRotatingFileHandler`类似，不过这个handler：
+    - 可以支持多进程
+    - 只支持自然日分割
+    - 暂不支持UTC
+    """
+
+    def __init__(self, filename, backupCount=0, encoding=None, delay=False, utc=False, **kwargs):
+        self.backup_count = backupCount
+        self.utc = utc
+        self.suffix = "%Y-%m-%d"
+        self.base_log_path = Path(filename)
+        self.base_filename = self.base_log_path.name
+        self.current_filename = self._compute_fn()
+        self.current_log_path = self._compute_lp()
+        BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
+
+    def shouldRollover(self, record):
+        """
+        判断是否该滚动日志，如果当前时间对应的日志文件名与当前打开的日志文件名不一致，则需要滚动日志
+        """
+        if self.current_filename != self._compute_fn():
+            # 日期变了，计算新的日志文件
+            self.current_filename = self._compute_fn()
+            return True
+        elif os.path.getsize(self.current_log_path) > 10485760:     # 判断文件是否大于10MB字节数
+            # 超过10M了，计算新的日志文件
+            seg = int(self.current_filename.split(".")[-2]) + 1
+            self.current_filename = self._compute_fn(seg=seg)
+            return True
+        return False
+
+    def doRollover(self):
+        """
+        滚动日志
+        """
+        # 关闭旧的日志文件
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+
+        # self.current_log_path = self.base_log_path.with_name(self.current_filename)
+        self.current_log_path = self._compute_lp()
+        # 打开新的日志文件
+        if not self.delay:
+            self.stream = self._open()
+
+        # 删除过期日志
+        # self.delete_expired_files()
+
+    def _compute_lp(self):
+        """
+        计算当前时间对应日志的路径
+        """
+        current_log_path = self.base_log_path.parent / time.strftime(self.suffix, time.localtime())
+        if not os.path.exists(current_log_path):
+            os.mkdir(current_log_path)
+        return current_log_path / self.current_filename
+
+    def _compute_fn(self, seg=0):
+        """
+        计算当前时间对应的日志文件名
+        """
+
+        return "limit-power" + "." + time.strftime(self.suffix, time.localtime()) + '.' + str(seg) +'.log'
+
+    def _open(self):
+        """
+        打开新的日志文件，同时更新base_filename指向的软链，修改软链不会对日志记录产生任何影响
+        """
+        if self.encoding is None:
+            stream = open(str(self.current_log_path), self.mode)
+        else:
+            stream = codecs.open(str(self.current_log_path), self.mode, self.encoding)
+
+        # # 删除旧的软链
+        # if self.base_log_path.exists():
+        #     try:
+        #         # 如果base_log_path不是软链或者指向的日志文件不对，则先删除该软链
+        #         if not self.base_log_path.is_symlink() or os.readlink(self.base_log_path) != self.current_log_path:
+        #             os.remove(self.base_log_path)
+        #     except OSError:
+        #         pass
+        #
+        # # 建立新的软链
+        # try:
+        #     os.symlink(self.current_log_path, str(self.base_log_path))
+        # except OSError:
+        #     pass
+        return stream
+
+    def delete_expired_files(self):
+        """
+        删除过期的日志
+        """
+        if self.backup_count <= 0:
+            return
+
+        file_names = os.listdir(str(self.base_log_path.parent))
+        result = []
+        prefix = self.base_filename + "."
+        plen = len(prefix)
+        for file_name in file_names:
+            if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", file_name):
+                result.append(file_name)
+        if len(result) < self.backup_count:
+            result = []
+        else:
+            result.sort()
+            result = result[:len(result) - self.backup_count]
+        import shutil
+        for file_name in result:
+            path = self.base_log_path.with_name(file_name)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+
+
+class Log(object):
+    def __init__(self):
+        # 定义对应的程序模块名name，默认为root
+        self.logger = logging.getLogger()
+        # 设置输出的等级
+        LEVELS = {'NOSET': logging.NOTSET,
+                  'DEBUG': logging.DEBUG,
+                  'INFO': logging.INFO,
+                  'WARNING': logging.WARNING,
+                  'ERROR': logging.ERROR,
+                  'CRITICAL': logging.CRITICAL}
+
+        # 必须设置，这里如果不显示设置，默认过滤掉warning之前的所有级别的信息
+        self.logger.setLevel(LEVELS['DEBUG'])
+
+        # 仅为matplotlib设置更高的日志等级（ERROR）
+        matplotlib_logger = logging.getLogger('matplotlib')
+        matplotlib_logger.setLevel(logging.ERROR)
+
+        # 日志输出格式
+        self.formatter = logging.Formatter(
+            '%(asctime)s - %(filename)s - %(levelname)s - %(message)s - %(funcName)s')  # 输出日志格式
+
+        # 创建一个handler， 向文件logname输出日志信息
+        # fh = logging.FileHandler(self.logname, 'a', encoding='utf-8')
+        # midnight:表示日志文件再每天半夜时分滚动
+        # interval: 间隔单位的个数，指等待多少个when的时间后 Logger会自动重建新闻继续进行日志记录
+        # backupCount:表示日志文件的保留个数，假如为30，保留最近30天的日志文件
+        # fh = logging.handlers.TimedRotatingFileHandler(self.getLogName(), when='midnight', interval=1, backupCount=30, encoding='utf-8')
+        # fh.suffix = "%Y-%m-%d"
+        # # fh.extMatch = r"^\d{4}-\d{2}-\d{2}"
+        # # 设置日志等级
+        # fh.setLevel(LEVELS['INFO'])
+        # # 设置handler的格式对象
+        # fh.setFormatter(self.formatter)
+        filename = self.getLogName()
+        dr_fh = DailyRotatingFileHandler(filename, backupCount=100, encoding='utf-8')
+        dr_fh.setFormatter(self.formatter)
+        # 将handler增加到logger中
+        self.logger.addHandler(dr_fh)
+
+        # 创建一个StreamHandler,用于输出到控制台
+        ch = logging.StreamHandler()
+        ch.setLevel(LEVELS['INFO'])
+        ch.setFormatter(self.formatter)
+        self.logger.addHandler(ch)
+
+        # # 关闭打开的文件
+        dr_fh.close()
+
+    def getLogName(self):
+        # log_path是存放日志的路径
+        # lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'logs'))
+        lib_path = Path(os.path.dirname(__file__)).parent / 'logs'
+        self.logger.info("日志输出路径为：{}".format(lib_path))
+        # 如果不存在这个logs文件夹，就自动创建一个
+        if not os.path.exists(lib_path):
+            os.mkdir(lib_path)
+        return lib_path / 'limit_power_link.log'
+
+
+
+if __name__ == "__main__":
+    logger = Log()
+    logger.info("this is info")
+    logger.debug("this is debug")
+    logger.error("this is error")
+    logger.warning("this is warning")
+    logger.critical("critical")