ok

Merge branch 'dev_awg' into dev_david

.gitignore

@@ -8,4 +8,4 @@ ipynb_checkpoints/**
 *$py.class
 *.zip
 **/.ipynb_checkpoints/
-.idea
+.idea

.idea/algorithm_platform.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="py312" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.12 (python3.12_env)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

common/database_dml.py

@@ -116,8 +116,9 @@ def get_data_fromMysql(params):
     #数据库读取实测气象
     engine = create_engine(f"mysql+pymysql://{mysql_conn}")
     # 定义SQL查询
-    env_df = pd.read_sql_query(query_sql, engine)
-    return env_df
+    with engine.connect() as conn:
+        df = pd.read_sql_query(query_sql, conn)
+    return df
 
 
 def insert_pickle_model_into_mongo(model, args):

evaluation_processing/analysis.py

@@ -135,6 +135,11 @@ def put_analysis_report_to_html(args, df_predict, df_accuracy):
             agg_dict['deviationAssessment'] = [np.nanmean, np.nansum]
             rename_cols.append('考核分数平均值')
             rename_cols.append('考核总分数')
+        if 'qualificationRate' in df_accuracy.columns:
+            agg_dict['qualificationRate'] = [np.nanmean]
+            rename_cols.append('合格率平均值')
+
+
         # 进行分组聚合，如果有需要聚合的列
         summary_df = df_accuracy.groupby('model').agg(agg_dict).reset_index()
         summary_df.columns = rename_cols

evaluation_processing/analysis_report.py

@@ -318,6 +318,9 @@ def put_analysis_report_to_html(args, df_clean, df_predict, df_accuracy):
             agg_dict['deviationAssessment'] = [np.nanmean, np.nansum]
             rename_cols.append('考核分数平均值')
             rename_cols.append('考核总分数')
+        if 'qualificationRate' in df_accuracy.columns:
+            agg_dict['qualificationRate'] = [np.nanmean]
+            rename_cols.append('合格率平均值')
         # 进行分组聚合，如果有需要聚合的列
         summary_df = df_accuracy.groupby('model').agg(agg_dict).reset_index()
         summary_df.columns = rename_cols

models_processing/model_predict/model_prediction_bp.py

@@ -12,7 +12,7 @@ app = Flask('model_prediction_bp——service')
 def model_prediction(df,args):
     if 'is_limit' in df.columns:
         df = df[df['is_limit'] == False]
-    features, time_steps, col_time, model_name,col_reserve =  str_to_list(args['features']), int(args['time_steps']),args['col_time'],args['model_name'],str_to_list(args['col_reserve'])
+    features, time_steps, col_time, model_name,col_reserve,howlongago =  str_to_list(args['features']), int(args['time_steps']),args['col_time'],args['model_name'],str_to_list(args['col_reserve']),int(args['howlongago'])
     feature_scaler,target_scaler = get_scaler_model_from_mongo(args)
     df = df.sort_values(by=col_time).fillna(method='ffill').fillna(method='bfill')
     scaled_features = feature_scaler.transform(df[features])
@@ -21,12 +21,14 @@ def model_prediction(df,args):
     # model = load_model(f'{farmId}_model.h5', custom_objects={'rmse': rmse})
     model = get_h5_model_from_mongo(args)
     y_predict = list(chain.from_iterable(target_scaler.inverse_transform([model.predict(scaled_features).flatten()])))
+
     result = df[-len(y_predict):]
     result['predict'] = y_predict
     result.loc[result['predict'] < 0, 'predict'] = 0
     result['model'] = model_name
-    features_reserve = col_reserve + ['model', 'predict']
-    return result[set(features_reserve)]
+    result['howlongago'] = howlongago
+    features_reserve = col_reserve + ['model', 'predict', 'howlongago']
+    return result[list(set(features_reserve))]
 
 
 @app.route('/model_prediction_bp', methods=['POST'])

models_processing/model_predict/model_prediction_lightgbm.py

@@ -73,7 +73,7 @@ def forecast_data_distribution(pre_data,args):
 
 
 def model_prediction(df,args):
-    mongodb_connection,mongodb_database,mongodb_model_table,model_name,col_reserve = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_model_table'],args['model_name'],str_to_list(args['col_reserve'])
+    mongodb_connection,mongodb_database,mongodb_model_table,model_name,howLongAgo,farm_id = "mongodb://root:sdhjfREWFWEF23e@192.168.1.43:30000/",args['mongodb_database'],args['mongodb_model_table'],args['model_name'],int(args['howLongAgo']),args['farm_id']
     client = MongoClient(mongodb_connection)
     db = client[mongodb_database]
     collection = db[mongodb_model_table]
@@ -85,12 +85,14 @@ def model_prediction(df,args):
         model_binary = model_data['model']  # 确保这个字段是存储模型的二进制数据
         # 反序列化模型 
         model = pickle.loads(model_binary)
-        df['predict'] = model.predict(df[model.feature_name()])
-        df.loc[df['predict']<0,'predict']=0
+        df['power_forecast'] = model.predict(df[model.feature_name()])
+        df.loc[df['power_forecast'] < 0, 'power_forecast'] = 0
         df['model'] = model_name
+        df['howLongAgo'] = howLongAgo
+        df['farm_id'] = farm_id
         print("model predict result  successfully!")
-    features_reserve = col_reserve + ['model','predict']
-    return df[set(features_reserve)]
+
+    return df[['dateTime','howLongAgo','model','farm_id','power_forecast','realPower']]
 
 
 @app.route('/model_prediction_lightgbm', methods=['POST'])

models_processing/model_predict/model_prediction_lstm.py

@@ -63,7 +63,7 @@ def forecast_data_distribution(pre_data,args):
 def model_prediction(df,args):
     if 'is_limit' in df.columns:
         df = df[df['is_limit'] == False]
-    features, time_steps, col_time, model_name,col_reserve =  str_to_list(args['features']), int(args['time_steps']),args['col_time'],args['model_name'],str_to_list(args['col_reserve'])
+    features, time_steps, col_time, model_name,col_reserve,howlongago = str_to_list(args['features']), int(args['time_steps']),args['col_time'],args['model_name'],str_to_list(args['col_reserve']),int(args['howlongago'])
     feature_scaler,target_scaler = get_scaler_model_from_mongo(args)
     df = df.sort_values(by=col_time).fillna(method='ffill').fillna(method='bfill')
     scaled_features = feature_scaler.transform(df[features])
@@ -72,12 +72,14 @@ def model_prediction(df,args):
     # model = load_model(f'{farmId}_model.h5', custom_objects={'rmse': rmse})
     model = get_h5_model_from_mongo(args)
     y_predict = list(chain.from_iterable(target_scaler.inverse_transform([model.predict(X_predict).flatten()])))
+
     result = df[-len(y_predict):]
     result['predict'] = y_predict
     result.loc[result['predict'] < 0, 'predict'] = 0
     result['model'] = model_name
-    features_reserve = col_reserve + ['model', 'predict']
-    return result[set(features_reserve)]
+    result['howlongago'] = howlongago
+    features_reserve = col_reserve + ['model', 'predict', 'howlongago']
+    return result[list(set(features_reserve))]
 
 
 @app.route('/model_prediction_lstm', methods=['POST'])

models_processing/model_predict/res_prediction.py

@@ -35,7 +35,7 @@ def model_prediction_lightgbm():
         power_df['model'] = args['model']
         power_df['predict'] = power_df[args['col_pre']]
         features_reserve = col_reserve + ['model', 'predict']
-        insert_data_into_mongo(power_df[set(features_reserve)], args)
+        insert_data_into_mongo(power_df[list(set(features_reserve))], args)
         success = 1
     except Exception as e:
         my_exception = traceback.format_exc()

requirements.txt

@@ -1,6 +1,6 @@
 pymongo==4.7.3
 pandas==2.2.3
-SQLAlchemy==1.4.46
+SQLAlchemy==2.0.39
 PyMySQL==1.1.1
 Flask==3.0.2
 waitress==2.1.2