Merge branch 'dev_david' of anweiguo/algorithm_platform into dev_awg

data_processing/data_operation/data_handler.py

@@ -104,8 +104,7 @@ class DataHandler(object):
 
     def fill_pre_data(self, unite):
         unite = unite.interpolate(method='linear')  # nwp先进行线性填充
-        unite = unite.fillna(method='ffill')  # 再对超过采样边缘无法填充的点进行二次填充
-        unite = unite.fillna(method='bfill')
+        unite = unite.ffill().bfill() # 再对超过采样边缘无法填充的点进行二次填充
         return unite
 
     def missing_time_splite(self, df, dt_short, dt_long, col_time):
@@ -197,7 +196,7 @@ class DataHandler(object):
         # 标准化特征和目标
         scaled_train_data = train_scaler.fit_transform(train_data_cleaned[self.opt.features])
         scaled_target = target_scaler.fit_transform(train_data_cleaned[[target]])
-        scaled_cap = target_scaler.transform(np.array([[self.opt.cap]]))[0,0]
+        scaled_cap = target_scaler.transform(np.array([[float(self.opt.cap)]]))[0,0]
         train_data_cleaned[self.opt.features] = scaled_train_data
         train_data_cleaned[[target]] = scaled_target
         # 3.缺值补值
@@ -233,13 +232,12 @@ class DataHandler(object):
         # features, time_steps, col_time, model_name, col_reserve = str_to_list(args['features']), int(
         #     args['time_steps']), args['col_time'], args['model_name'], str_to_list(args['col_reserve'])
         col_time, features = self.opt.col_time, self.opt.features
-        data = data.applymap(lambda x: float(x.to_decimal()) if isinstance(x, Decimal128) else float(x) if isinstance(x, numbers.Number) else x)
+        data = data.map(lambda x: float(x.to_decimal()) if isinstance(x, Decimal128) else float(x) if isinstance(x, numbers.Number) else x)
         data = data.sort_values(by=col_time).reset_index(drop=True, inplace=False)
+        pre_data = data[features].copy()
         if self.opt.Model['predict_data_fill']:
-            data = self.fill_pre_data(data)
-        pre_data = data[features]
-        scaled_features = feature_scaler.transform(data[features])
-        pre_data.loc[:, features] = scaled_features
+            pre_data = self.fill_pre_data(pre_data)
+        pre_data.loc[:, features] = feature_scaler.transform(pre_data)
         if bp_data:
             pre_x = np.array(pre_data)
         else:

models_processing/model_tf/test.yaml

@@ -0,0 +1,92 @@
+Model:
+  add_train: false
+  batch_size: 64
+  dropout_rate: 0.2
+  epoch: 200
+  fusion: true
+  hidden_size: 64
+  his_points: 16
+  how_long_fill: 10
+  input_size: 24
+  lambda_value_1: 0.02
+  lambda_value_2: 0.01
+  learning_rate: 0.001
+  lstm_layers: 1
+  output_size: 16
+  patience: 10
+  predict_data_fill: true
+  shuffle_train_data: false
+  test_data_fill: false
+  time_step: 16
+  train_data_fill: true
+  use_cuda: false
+  valid_data_rate: 0.15
+region: south
+calculate: []
+cap: 153.0
+dataloc: ./data
+env_columns:
+- C_TIME
+- C_CELLT
+- C_DIFFUSER
+- C_GLOBALR
+- C_RH
+- C_REAL_VALUE
+full_field: true
+history_hours: 1
+new_field: true
+features:
+- time
+- temperature10
+- temperature190
+- direction160
+- direction40
+- temperature110
+- direction80
+- speed60
+- mcc
+- temperature150
+- speed20
+- speed110
+- direction120
+- speed190
+- solarZenith
+- temperature90
+- direction200
+- speed150
+- temperature50
+- direction30
+- temperature160
+- direction170
+- temperature20
+- direction70
+- direction130
+- temperature200
+- speed70
+- temperature120
+- speed30
+- speed100
+- speed80
+- speed180
+- dniCalcd
+- speed140
+- temperature60
+- dateTime
+- temperature30
+- temperature170
+- direction20
+- humidity2
+- direction180
+- realPowerAvg
+- direction60
+- direction140
+- speed40
+- hcc
+target: realPower
+repair_days: 81
+repair_model_cycle: 5
+spot_trading: []
+update_add_train_days: 60
+update_coe_days: 3
+version: solar-3.1.0.south
+

models_processing/model_tf/tf_bp_pre.py

@@ -53,7 +53,7 @@ def model_prediction_bp():
         # ------------ 获取数据，预处理预测数据------------
         pre_data = get_data_from_mongo(args)
         feature_scaler, target_scaler = get_scaler_model_from_mongo(args)
-        bp.opt.cap = round(target_scaler.transform(np.array([[args['cap']]]))[0, 0], 2)
+        bp.opt.cap = round(target_scaler.transform(np.array([[float(args['cap'])]]))[0, 0], 2)
         # ------------ 获取模型，预测结果------------
         bp.get_model(args)
         dh.opt.features = json.loads(bp.model_params).get('features', args['features'])
@@ -71,17 +71,13 @@ def model_prediction_bp():
                 pre_data['howLongAgo'] = int(args['howLongAgo'])
                 res_cols += ['howLongAgo']
         else:
-            pre_data['cdq'] = args.get('cdq', 1)
-            pre_data['dq'] = args.get('dq', 1)
-            pre_data['zq'] = args.get('zq', 1)
             pre_data['power_forecast'] = res[:len(pre_data)]
             pre_data.rename(columns={args['col_time']: 'date_time'}, inplace=True)
-            res_cols = ['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']
+            res_cols = ['date_time', 'power_forecast', 'farm_id']
 
         pre_data = pre_data[res_cols]
-
         pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
-        pre_data.loc[pre_data['power_forecast'] > args['cap'], 'power_forecast'] = args['cap']
+        pre_data.loc[pre_data['power_forecast'] > float(args['cap']), 'power_forecast'] = float(args['cap'])
         pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
 
         insert_data_into_mongo(pre_data, args)

models_processing/model_tf/tf_cnn_pre.py

@@ -53,7 +53,7 @@ def model_prediction_bp():
     try:
         pre_data = get_data_from_mongo(args)
         feature_scaler, target_scaler = get_scaler_model_from_mongo(args)
-        cnn.opt.cap = round(target_scaler.transform(np.array([[args['cap']]]))[0, 0], 2)
+        cnn.opt.cap = round(target_scaler.transform(np.array([[float(args['cap'])]]))[0, 0], 2)
 
         cnn.get_model(args)
         dh.opt.features = json.loads(cnn.model_params).get('features', args['features'])
@@ -72,16 +72,13 @@ def model_prediction_bp():
                 pre_data['howLongAgo'] = int(args['howLongAgo'])
                 res_cols += ['howLongAgo']
         else:
-            pre_data['cdq'] = args.get('cdq', 1)
-            pre_data['dq'] = args.get('dq', 1)
-            pre_data['zq'] = args.get('zq', 1)
             pre_data['power_forecast'] = res[:len(pre_data)]
             pre_data.rename(columns={args['col_time']: 'date_time'}, inplace=True)
-            res_cols = ['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']
+            res_cols = ['date_time', 'power_forecast', 'farm_id']
         pre_data = pre_data[res_cols]
 
         pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
-        pre_data.loc[pre_data['power_forecast'] > args['cap'], 'power_forecast'] = args['cap']
+        pre_data.loc[pre_data['power_forecast'] > float(args['cap']), 'power_forecast'] = float(args['cap'])
         pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
 
         insert_data_into_mongo(pre_data, args)

models_processing/model_tf/tf_lstm_pre.py

@@ -53,7 +53,7 @@ def model_prediction_bp():
     try:
         pre_data = get_data_from_mongo(args)
         feature_scaler, target_scaler = get_scaler_model_from_mongo(args)
-        ts.opt.cap = round(target_scaler.transform(np.array([[args['cap']]]))[0, 0], 2)
+        ts.opt.cap = round(target_scaler.transform(np.array([[float(args['cap'])]]))[0, 0], 2)
         ts.get_model(args)
         dh.opt.features = json.loads(ts.model_params).get('features', args['features'])
         scaled_pre_x, pre_data = dh.pre_data_handler(pre_data, feature_scaler)
@@ -69,16 +69,13 @@ def model_prediction_bp():
                 pre_data['howLongAgo'] = int(args['howLongAgo'])
                 res_cols += ['howLongAgo']
         else:
-            pre_data['cdq'] = args.get('cdq', 1)
-            pre_data['dq'] = args.get('dq', 1)
-            pre_data['zq'] = args.get('zq', 1)
             pre_data['power_forecast'] = res[:len(pre_data)]
             pre_data.rename(columns={args['col_time']: 'date_time'}, inplace=True)
-            res_cols = ['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']
+            res_cols = ['date_time', 'power_forecast', 'farm_id']
         pre_data = pre_data[res_cols]
 
         pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
-        pre_data.loc[pre_data['power_forecast'] > args['cap'], 'power_forecast'] = args['cap']
+        pre_data.loc[pre_data['power_forecast'] > float(args['cap']), 'power_forecast'] = float(args['cap'])
         pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
 
         insert_data_into_mongo(pre_data, args)

models_processing/model_tf/tf_test.py

@@ -92,7 +92,7 @@ class TSHandler(object):
         x = Dropout(0.5)(x)
 
         # %% 输出层（可扩展为概率预测）
-        output = Dense(1, activation='linear', name='main_output')(x)
+        output = Dense(16, activation='linear', name='main_output')(x)
 
         # 概率预测扩展（可选）
         # variance = Dense(1, activation='softplus')(x)  # 输出方差

models_processing/model_tf/tf_test_pre.py

@@ -25,7 +25,7 @@ app = Flask('tf_test_pre——service')
 
 with app.app_context():
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    with open(os.path.join(current_dir, 'lstm.yaml'), 'r', encoding='utf-8') as f:
+    with open(os.path.join(current_dir, 'test.yaml'), 'r', encoding='utf-8') as f:
         args = yaml.safe_load(f)
 
     dh = DataHandler(logger, args)
@@ -53,7 +53,7 @@ def model_prediction_test():
     try:
         pre_data = get_data_from_mongo(args)
         feature_scaler, target_scaler = get_scaler_model_from_mongo(args)
-        ts.opt.cap = round(target_scaler.transform(np.array([[args['cap']]]))[0, 0], 2)
+        ts.opt.cap = round(target_scaler.transform(np.array([[float(args['cap'])]]))[0, 0], 2)
 
         ts.get_model(args)
         dh.opt.features = json.loads(ts.model_params).get('features', args['features'])
@@ -71,18 +71,13 @@ def model_prediction_test():
                 pre_data['howLongAgo'] = int(args['howLongAgo'])
                 res_cols += ['howLongAgo']
         else:
-            pre_data['cdq'] = args.get('cdq', 1)
-            pre_data['dq'] = args.get('dq', 1)
-            pre_data['zq'] = args.get('zq', 1)
             pre_data['power_forecast'] = res[:len(pre_data)]
             pre_data.rename(columns={args['col_time']: 'date_time'}, inplace=True)
-            res_cols = ['date_time', 'power_forecast', 'farm_id', 'cdq', 'dq', 'zq']
+            res_cols = ['date_time', 'power_forecast', 'farm_id']
         pre_data = pre_data[res_cols]
-
         pre_data['power_forecast'] = pre_data['power_forecast'].round(2)
-        pre_data.loc[pre_data['power_forecast'] > args['cap'], 'power_forecast'] = args['cap']
+        pre_data.loc[pre_data['power_forecast'] > float(args['cap']), 'power_forecast'] = float(args['cap'])
         pre_data.loc[pre_data['power_forecast'] < 0, 'power_forecast'] = 0
-
         insert_data_into_mongo(pre_data, args)
         success = 1
     except Exception as e:

models_processing/model_tf/tf_test_train.py

@@ -20,7 +20,7 @@ app = Flask('tf_test_train——service')
 
 with app.app_context():
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    with open(os.path.join(current_dir, 'lstm.yaml'), 'r', encoding='utf-8') as f:
+    with open(os.path.join(current_dir, 'test.yaml'), 'r', encoding='utf-8') as f:
         args = yaml.safe_load(f)
 
     dh = DataHandler(logger, args)