Merge branch 'dev_david' of anweiguo/algorithm_platform into dev_awg

data_processing/data_operation/pre_prod_ftp.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# @FileName  :pre_prod_ftp.py
+# @Time      :2025/3/4 13:02
+# @Author    :David
+# @Company: shenyang JY
+
+"""
+要实现的功能：
+1.
+"""
+
+if __name__ == "__main__":
+    run_code = 0

models_processing/model_koi/tf_bp_pre.py

@@ -89,7 +89,7 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    serve(app, host="0.0.0.0", port=1010, threads=4)
+    serve(app, host="0.0.0.0", port=10110, threads=4)
     print("server start!")
 
     # ------------------------测试代码------------------------

models_processing/model_koi/tf_bp_train.py

@@ -80,7 +80,7 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    serve(app, host="0.0.0.0", port=10103, threads=4)
+    serve(app, host="0.0.0.0", port=10111, threads=4)
     # print("server start!")
     # args_dict = {"mongodb_database": 'david_test', 'scaler_table': 'j00083_scaler', 'model_name': 'bp1.0.test',
     # 'model_table': 'j00083_model', 'mongodb_read_table': 'j00083', 'col_time': 'dateTime',

models_processing/model_koi/tf_cnn_pre.py

@@ -90,7 +90,7 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    serve(app, host="0.0.0.0", port=1010, threads=4)
+    serve(app, host="0.0.0.0", port=10112, threads=4)
     print("server start!")
 
     # ------------------------测试代码------------------------

models_processing/model_koi/tf_cnn_train.py

@@ -83,7 +83,7 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    serve(app, host="0.0.0.0", port=10103, threads=4)
+    serve(app, host="0.0.0.0", port=10113, threads=4)
     # print("server start!")
     # args_dict = {"mongodb_database": 'david_test', 'scaler_table': 'j00083_scaler', 'model_name': 'bp1.0.test',
     # 'model_table': 'j00083_model', 'mongodb_read_table': 'j00083', 'col_time': 'dateTime',

models_processing/model_koi/tf_lstm.py

@@ -43,7 +43,7 @@ class TSHandler(object):
 
         con1 = Conv1D(filters=64, kernel_size=5, strides=1, padding='valid', activation='relu', kernel_regularizer=l2_reg)(nwp_input)
         con1_p = MaxPooling1D(pool_size=5, strides=1, padding='valid', data_format='channels_last')(con1)
-        nwp_lstm = LSTM(units=opt.Model['hidden_size'], return_sequences=False, kernel_regularizer=l2_reg)(nwp_input)
+        nwp_lstm = LSTM(units=opt.Model['hidden_size'], return_sequences=False, kernel_regularizer=l2_reg)(con1_p)
 
         output = Dense(opt.Model['output_size'], name='cdq_output')(nwp_lstm)
 

models_processing/model_koi/tf_lstm_pre.py

@@ -90,36 +90,36 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    # serve(app, host="0.0.0.0", port=1010x, threads=4)
+    serve(app, host="0.0.0.0", port=10114, threads=4)
     print("server start!")
 
     # ------------------------测试代码------------------------
-    args_dict = {"mongodb_database": 'david_test', 'scaler_table': 'j00083_scaler', 'model_name': 'bp1.0.test',
-                 'model_table': 'j00083_model', 'mongodb_read_table': 'j00083_test', 'col_time': 'date_time', 'mongodb_write_table': 'j00083_rs',
-                 'features': 'speed10,direction10,speed30,direction30,speed50,direction50,speed70,direction70,speed90,direction90,speed110,direction110,speed150,direction150,speed170,direction170'}
-    args_dict['features'] = args_dict['features'].split(',')
-    arguments.update(args_dict)
-    dh = DataHandler(logger, arguments)
-    ts = TSHandler(logger)
-    opt = argparse.Namespace(**arguments)
-
-    opt.Model['input_size'] = len(opt.features)
-    pre_data = get_data_from_mongo(args_dict)
-    feature_scaler, target_scaler = get_scaler_model_from_mongo(arguments)
-    pre_x = dh.pre_data_handler(pre_data, feature_scaler, opt)
-    ts.get_model(arguments)
-    result = ts.predict(pre_x)
-    result1 = list(chain.from_iterable(target_scaler.inverse_transform([result.flatten()])))
-    pre_data['power_forecast'] = result1[:len(pre_data)]
-    pre_data['farm_id'] = 'J00083'
-    pre_data['cdq'] = 1
-    pre_data['dq'] = 1
-    pre_data['zq'] = 1
-    pre_data.rename(columns={arguments['col_time']: 'date_time'}, inplace=True)
-    pre_data = pre_data[['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']]
-
-    pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
-    pre_data.loc[pre_data['power_forecast'] > opt.cap, 'power_forecast'] = opt.cap
-    pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
-
-    insert_data_into_mongo(pre_data, arguments)
+    # args_dict = {"mongodb_database": 'david_test', 'scaler_table': 'j00083_scaler', 'model_name': 'bp1.0.test',
+    #              'model_table': 'j00083_model', 'mongodb_read_table': 'j00083_test', 'col_time': 'date_time', 'mongodb_write_table': 'j00083_rs',
+    #              'features': 'speed10,direction10,speed30,direction30,speed50,direction50,speed70,direction70,speed90,direction90,speed110,direction110,speed150,direction150,speed170,direction170'}
+    # args_dict['features'] = args_dict['features'].split(',')
+    # arguments.update(args_dict)
+    # dh = DataHandler(logger, arguments)
+    # ts = TSHandler(logger)
+    # opt = argparse.Namespace(**arguments)
+    #
+    # opt.Model['input_size'] = len(opt.features)
+    # pre_data = get_data_from_mongo(args_dict)
+    # feature_scaler, target_scaler = get_scaler_model_from_mongo(arguments)
+    # pre_x = dh.pre_data_handler(pre_data, feature_scaler, opt)
+    # ts.get_model(arguments)
+    # result = ts.predict(pre_x)
+    # result1 = list(chain.from_iterable(target_scaler.inverse_transform([result.flatten()])))
+    # pre_data['power_forecast'] = result1[:len(pre_data)]
+    # pre_data['farm_id'] = 'J00083'
+    # pre_data['cdq'] = 1
+    # pre_data['dq'] = 1
+    # pre_data['zq'] = 1
+    # pre_data.rename(columns={arguments['col_time']: 'date_time'}, inplace=True)
+    # pre_data = pre_data[['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']]
+    #
+    # pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
+    # pre_data.loc[pre_data['power_forecast'] > opt.cap, 'power_forecast'] = opt.cap
+    # pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
+    #
+    # insert_data_into_mongo(pre_data, arguments)

models_processing/model_koi/tf_lstm_train.py

@@ -85,23 +85,23 @@ if __name__ == "__main__":
     logger = logging.getLogger("model_training_bp log")
     from waitress import serve
 
-    serve(app, host="0.0.0.0", port=10103, threads=4)
+    serve(app, host="0.0.0.0", port=10115, threads=4)
     print("server start!")
-    # args_dict = {"mongodb_database": 'david_test', 'scaler_table': 'j00083_scaler', 'model_name': 'bp1.0.test',
-    # 'model_table': 'j00083_model', 'mongodb_read_table': 'j00083', 'col_time': 'dateTime',
+    # args_dict = {"mongodb_database": 'realtimeDq', 'scaler_table': 'j00600_scaler', 'model_name': 'lstm1',
+    # 'model_table': 'j00600_model', 'mongodb_read_table': 'j00600', 'col_time': 'dateTime',
     # 'features': 'speed10,direction10,speed30,direction30,speed50,direction50,speed70,direction70,speed90,direction90,speed110,direction110,speed150,direction150,speed170,direction170'}
     # args_dict['features'] = args_dict['features'].split(',')
-    # arguments.update(args_dict)
-    # dh = DataHandler(logger, arguments)
-    # ts = TSHandler(logger)
-    # opt = argparse.Namespace(**arguments)
+    # args.update(args_dict)
+    # dh = DataHandler(logger, args)
+    # ts = TSHandler(logger, args)
+    # opt = argparse.Namespace(**args)
     # opt.Model['input_size'] = len(opt.features)
     # train_data = get_data_from_mongo(args_dict)
-    # train_x, valid_x, train_y, valid_y, scaled_train_bytes, scaled_target_bytes = dh.train_data_handler(train_data, opt)
-    # ts_model = ts.training(opt, [train_x, train_y, valid_x, valid_y])
+    # train_x, train_y, valid_x, valid_y, scaled_train_bytes, scaled_target_bytes = dh.train_data_handler(train_data)
+    # ts_model = ts.training([train_x, train_y, valid_x, valid_y])
     #
     # args_dict['gen_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
-    # args_dict['params'] = arguments
+    # args_dict['params'] = args
     # args_dict['descr'] = '测试'
     # insert_trained_model_into_mongo(ts_model, args_dict)
     # insert_scaler_model_into_mongo(scaled_train_bytes, scaled_target_bytes, args_dict)

run_all.py

@@ -16,6 +16,14 @@ services = [
     ("models_processing/model_predict/model_prediction_lightgbm.py", 10090),
     ("models_processing/model_train/model_training_lstm.py", 10096),
     ("models_processing/model_predict/model_prediction_lstm.py", 10097),
+
+    ("models_processing/model_koi/tf_bp_pre.py", 10110),
+    ("models_processing/model_koi/tf_bp_train.py", 10111),
+    ("models_processing/model_koi/tf_cnn_pre.py", 10112),
+    ("models_processing/model_koi/tf_cnn_train.py", 10113),
+    ("models_processing/model_koi/tf_lstm_pre.py", 10114),
+    ("models_processing/model_koi/tf_lstm_train.py", 10115),
+
     ("post_processing/post_processing.py", 10098),
     ("evaluation_processing/analysis.py", 10099),
     ("models_processing/model_predict/res_prediction.py", 10105),