{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "660798fa-cf08-4cfd-b553-c6d750d7e7ad", "metadata": {}, "outputs": [], "source": [ "import algo_test" ] }, { "cell_type": "code", "execution_count": 11, "id": "ff42670b-301d-4ee1-8da2-6a658fc5f286", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'_id': ObjectId('66ebba82a3b7dbe82538e34a'), 'name': 'Alice', 'age': 25, 'city': 'Beijing'}\n", "\n" ] } ], "source": [ "import subprocess\n", "\n", "# 使用 subprocess 执行 .py 文件\n", "result = subprocess.run(['python', 'algo_test.py'], capture_output=True, text=True)\n", "\n", "# 打印输出\n", "print(result.stdout)" ] }, { "cell_type": "code", "execution_count": 3, "id": "b47d615a-5953-43c0-9289-387d5c823a7a", "metadata": {}, "outputs": [], "source": [ "!pip freeze > requirements.txt" ] }, { "cell_type": "code", "execution_count": 5, "id": "752c3ab3-cdef-47b0-9d9a-805348d30d6e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3.9.19 (main, May 6 2024, 20:12:36) [MSC v.1916 64 bit (AMD64)]\n" ] } ], "source": [ "parser.add_argument('--mongodb_connection', type=str, required=True, help=\"mongodb连接\")# algorithm_platform\n", "parser.add_argument('--mongodb_database', type=str, required=True, help=\"mongodb数据库名\")# algorithm_platform\n", "parser.add_argument('--mongodb_read_table', type=str, required=True, help=\"mongodb数据库读取表\")# tmp_one\n", "parser.add_argument('--mongodb_model_table', type=str, required=True, help=\"mongodb读取模型集合\")# tmp_one\n", "parser.add_argument('--model_name', type=str, required=True, help=\"保存模型名\")# tmp_one\n", "#出参\n", "parser.add_argument('--mongodb_write_table', type=str, required=True, help=\"mongodb数据库存储集合\")# tmp_one" ] }, { "cell_type": "code", "execution_count": 9, "id": "acc6aa2a-0f1d-43f3-985d-cd37c4a5ac6e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Output:\n", "Program starts execution!\n", "args: Namespace(mongodb_connection='mongodb://root:Jydl3377@192.168.1.33:27017', mongodb_database='algorithm_platform', mongodb_read_table='J00260NWP_records', mongodb_model_table='models', model_name='lightgbm_model', mongodb_write_table='J00260NWP_records_predict')\n", "model predict result successfully!\n", "data inserted successfully!\n", "Program execution ends!\n", "\n", "Error:\n", "\n" ] } ], "source": [ "import subprocess\n", "\n", "# 构建命令\n", "command = [\n", " \"python\", \"model_prediction.py\",\n", " \"--mongodb_connection\", \"mongodb://root:Jydl3377@192.168.1.33:27017\",\n", " \"--mongodb_database\", \"algorithm_platform\",\n", " \"--mongodb_read_table\", \"J00260NWP_records\",\n", " \"--mongodb_model_table\", \"models\",\n", " \"--model_name\",\"lightgbm_model\",\n", " \"--mongodb_write_table\", \"J00260NWP_records_predict\",\n", "]\n", "\n", "# 使用 subprocess.run 来执行命令\n", "result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n", "\n", "# 输出结果和错误信息\n", "print(\"Output:\")\n", "print(result.stdout)\n", "print(\"Error:\")\n", "print(result.stderr)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "14a9feb4-5625-45d7-8e0f-6a8dea8c9f24", "metadata": {}, "outputs": [], "source": [ "docker run model_prediction_lightgbm --mongodb_connection \"mongodb://root:Jydl3377@192.168.1.33:27017\" --mongodb_database \"algorithm_platform\" --mongodb_read_table \"J00260NWP_records\" --mongodb_model_table \"models\" --model_name \"lightgbm_model\" --mongodb_database \"algorithm_platform\" --mongodb_write_table \"J00260NWP_records_predict\" " ] }, { "cell_type": "code", "execution_count": null, "id": "c0d791f8-12d8-4ab9-b561-b474e2a739f0", "metadata": {}, "outputs": [], "source": [ "docker run lightgbm_model.py --numerical_features ','.join(['NWP预测总辐射', 'NWP-10米预测风速', 'NWP-10米预测风向', 'NWP-30米预测风速', 'NWP-30米预测风向','NWP-50米预测风速', 'NWP-50米预测风向', 'NWP-70米预测风速', 'NWP-70米预测风向','NWP-90米预测风速', 'NWP-90米预测风向', 'NWP-110米预测风速', 'NWP-110米预测风向','NWP-150米预测风速', 'NWP-150米预测风向', 'NWP-170米预测风速', 'NWP-170米预测风向','NWP-低云量', 'NWP-中云量', 'NWP-高云量', 'NWP-总云量', 'NWP-晴空总辐射', 'NWP-法相直辐射','NWP-总降水强度', 'NWP-温度', 'NWP-湿度', 'NWP-气压']) --categorical_features ','.join([]) --label \"实际功率\" --learning_rate \"0.01\" --num_leaves \"31\" --min_data_in_leaf \"21\" --mongodb_connection \"mongodb://root:Jydl3377@192.168.1.33:27017\" --mongodb_database \"algorithm_platform\" --mongodb_read_table \"J00260NWP_records\" --mongodb_write_table \"models\" --model_name \"lightgbm_model\"" ] }, { "cell_type": "code", "execution_count": null, "id": "6d13761d-3ba7-441c-903d-db293ab23223", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2024-09-26 11:55:37,911 - waitress - INFO - Serving on http://0.0.0.0:10089\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Program starts execution!\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2024-09-26 11:57:14,162 - model_prediction_lightgbm log - INFO - {'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'J00260NWP_records_predict', 'mongodb_model_table': 'models', 'model_name': 'lightgbm_model'}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Program starts execution!\n", "args {'mongodb_connection': 'mongodb://root:Jydl3377@192.168.1.33:27017', 'mongodb_database': 'algorithm_platform', 'mongodb_read_table': 'J00260NWP_records', 'mongodb_write_table': 'J00260NWP_records_predict', 'mongodb_model_table': 'models', 'model_name': 'lightgbm_model'}\n", "model predict result successfully!\n", "Collection 'J00260NWP_records_predict already exist, deleted successfully!\n", "data inserted successfully!\n", "Program execution ends!\n" ] } ], "source": [ " %run model_prediction_lightgbm.py" ] }, { "cell_type": "code", "execution_count": 2, "id": "9ca17599-e453-479c-a9b3-b1d15f2d54de", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import lightgbm as lgb\n", "import argparse\n", "import pandas as pd \n", "import numpy as np\n", "from pymongo import MongoClient\n", "import pickle\n", "client = MongoClient(\"mongodb://root:Jydl3377@192.168.1.33:27017\")\n", "db = client[\"algorithm_platform\"]\n", "collection = db[\"models\"]\n", "model_data = collection.find_one({\"model_name\": \"lightgbm_model\"})\n", "if model_data is not None:\n", " # 假设模型存储在字段 'model' 中\n", " model_binary = model_data['model'] # 确保这个字段是存储模型的二进制数据\n", "\n", " # 反序列化模型\n", " model = pickle.loads(model_binary)\n", "model" ] }, { "cell_type": "code", "execution_count": 10, "id": "ff940fec-55fd-4cbd-99ef-5564c6632efb", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
_idC_TIME理论功率第1个设备轮毂风速实际功率实时环境数据预测功率NWP预测总辐射NWP-10米预测风速NWP-10米预测风向...NWP-中云量NWP-高云量NWP-总云量NWP-晴空总辐射NWP-法相直辐射NWP-总降水强度NWP-温度NWP-湿度NWP-气压predict
066ecc86f1075358d129a1d562024-05-02 22:30:00130.059.63104.76-0.9992.520.03.6477202.7945...0.017320.291820.39270.00.00.014.081823.4918892.332777.728147
166ecc86f1075358d129a1d572024-05-02 22:45:00130.519.72104.76-0.9993.240.03.9228202.6482...0.019120.300920.38640.00.00.013.826424.0436892.380984.568199
\n", "

2 rows × 35 columns

\n", "
" ], "text/plain": [ " _id C_TIME 理论功率 第1个设备轮毂风速 实际功率 \\\n", "0 66ecc86f1075358d129a1d56 2024-05-02 22:30:00 130.05 9.63 104.76 \n", "1 66ecc86f1075358d129a1d57 2024-05-02 22:45:00 130.51 9.72 104.76 \n", "\n", " 实时环境数据 预测功率 NWP预测总辐射 NWP-10米预测风速 NWP-10米预测风向 ... NWP-中云量 NWP-高云量 \\\n", "0 -0.99 92.52 0.0 3.6477 202.7945 ... 0.0173 20.2918 \n", "1 -0.99 93.24 0.0 3.9228 202.6482 ... 0.0191 20.3009 \n", "\n", " NWP-总云量 NWP-晴空总辐射 NWP-法相直辐射 NWP-总降水强度 NWP-温度 NWP-湿度 NWP-气压 \\\n", "0 20.3927 0.0 0.0 0.0 14.0818 23.4918 892.3327 \n", "1 20.3864 0.0 0.0 0.0 13.8264 24.0436 892.3809 \n", "\n", " predict \n", "0 77.728147 \n", "1 84.568199 \n", "\n", "[2 rows x 35 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "collection = db[\"J00260NWP_records_predict\"] # 集合名称\n", "data_from_db = collection.find() # 这会返回一个游标（cursor）\n", "# 将游标转换为列表，并创建 pandas DataFrame\n", "df = pd.DataFrame(list(data_from_db))\n", "df.head(2)" ] }, { "cell_type": "code", "execution_count": 5, "id": "7feaff6d-e41a-4bd7-b6fb-a63fc6900cca", "metadata": {}, "outputs": [], "source": [ "df['predict'] = model.predict(df[model.feature_name()])" ] }, { "cell_type": "code", "execution_count": 6, "id": "8a22f3e9-feff-438e-81e2-b5ceaa15ac48", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
_idC_TIME理论功率第1个设备轮毂风速实际功率实时环境数据预测功率NWP预测总辐射NWP-10米预测风速NWP-10米预测风向...NWP-中云量NWP-高云量NWP-总云量NWP-晴空总辐射NWP-法相直辐射NWP-总降水强度NWP-温度NWP-湿度NWP-气压predict
066ecc86f1075358d129a1d562024-05-02 22:30:00130.059.63104.76-0.9992.520.03.6477202.7945...0.017320.291820.39270.00.00.014.081823.4918892.332777.728147
166ecc86f1075358d129a1d572024-05-02 22:45:00130.519.72104.76-0.9993.240.03.9228202.6482...0.019120.300920.38640.00.00.013.826424.0436892.380984.568199
\n", "

2 rows × 35 columns

\n", "
" ], "text/plain": [ " _id C_TIME 理论功率 第1个设备轮毂风速 实际功率 \\\n", "0 66ecc86f1075358d129a1d56 2024-05-02 22:30:00 130.05 9.63 104.76 \n", "1 66ecc86f1075358d129a1d57 2024-05-02 22:45:00 130.51 9.72 104.76 \n", "\n", " 实时环境数据 预测功率 NWP预测总辐射 NWP-10米预测风速 NWP-10米预测风向 ... NWP-中云量 NWP-高云量 \\\n", "0 -0.99 92.52 0.0 3.6477 202.7945 ... 0.0173 20.2918 \n", "1 -0.99 93.24 0.0 3.9228 202.6482 ... 0.0191 20.3009 \n", "\n", " NWP-总云量 NWP-晴空总辐射 NWP-法相直辐射 NWP-总降水强度 NWP-温度 NWP-湿度 NWP-气压 \\\n", "0 20.3927 0.0 0.0 0.0 14.0818 23.4918 892.3327 \n", "1 20.3864 0.0 0.0 0.0 13.8264 24.0436 892.3809 \n", "\n", " predict \n", "0 77.728147 \n", "1 84.568199 \n", "\n", "[2 rows x 35 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head(2)" ] } ], "metadata": { "kernelspec": { "display_name": "Python (panddlets)", "language": "python", "name": "panddlets" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.19" } }, "nbformat": 4, "nbformat_minor": 5 }