1.0

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/.gitignore

@@ -1,13 +0,0 @@
-*/__pycache__
-/__pycache__
-/.idea
-/checkpoint
-/log
-/data
-/figure
-*.log
-*.swp
-/log
-/data
-
-

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/Readme.md

@@ -1,18 +0,0 @@
-## 超短期功率预测系统训练端
-
-这个项目将LSTM长短期时序模型用于超短期电力功率预测任务，实现特性如下: 
-
-- 程序简洁、模块化
-- 支持可扩展的Keras框架（LSTM，可修改网络层）
-- 参数、模型和框架支持高度可定制和修改
-- 支持增量训练（在预训练模型上进行微调）
-- 支持同时预测多个指标（目前预测实际功率）
-- 支持预测任意时间节点数（目前设置16个点）
-- 支持训练可视化和记录日志
-
-
-
-| 训练case | 表头  |
-|--------| ----  |
-| 1      | 单元格 |
-| 2      | 单元格 |

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/back.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/4/14 15:32
-# file: back.py
-# author: David
-# company: shenyang JY
-import sys
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-
-
-class data_analyse(object):
-    def __init__(self, opt, logger, process):
-        self.opt = opt
-        self.logger = logger
-        self.ds = process
-
-    def calculate_acc(self, label_data, predict_data):
-        loss = np.sum((label_data - predict_data) ** 2) / len(label_data)  # mse
-        loss_sqrt = np.sqrt(loss)  # rmse
-        loss_acc = 1 - loss_sqrt / self.opt.cap
-        return loss_acc
-
-    def get_16_points(self, results):
-        # results为模型预测的一维数组，遍历，取每16个点的最后一个点
-        preds = []
-        for res in results:
-            preds.append(res.iloc[-1].values)
-        return np.array(preds)
-
-    def predict_acc(self, predict_data, dfy):
-        predict_data = predict_data * self.ds.std['C_REAL_VALUE'] + self.ds.mean['C_REAL_VALUE']
-        dfs = dfy[0]
-        for i in range(1, len(dfy)):
-            dfs.extend(dfy[i])
-        for i, df in enumerate(dfs):
-            df["PREDICT"] = predict_data[i]
-            dfs[i] = df
-        data = self.get_16_points(dfs)
-        df = pd.DataFrame(data, columns=['C_TIME', 'C_REAL_VALUE', 'C_FP_VALUE', 'PREDICT'])
-        # label_data = label_data.reshape((-1, self.opt.output_size))
-        # label_data 要进行反归一化
-
-        label_name = [self.opt.feature_columns[i] for i in self.opt.label_in_feature_index]
-        loss_norm = self.calculate_acc(df['C_REAL_VALUE'], df['PREDICT'])
-
-        self.logger.info("The mean squared error of power {} is ".format(label_name) + str(loss_norm))
-
-        loss_norm = self.calculate_acc(df['C_REAL_VALUE'], df['C_FP_VALUE'])
-
-        self.logger.info("The mean squared error of power {} is ".format(label_name) + str(loss_norm))
-        self.preidct_draw(df['C_REAL_VALUE'].values, df['PREDICT'].values)
-
-    def preidct_draw(self, label_data, predict_data):
-        X = list(range(label_data.shape[0]))
-        print("label_x = ", X)
-        label_column_num = len(self.opt.label_columns)
-        label_name = [self.opt.feature_columns[i] for i in self.opt.label_in_feature_index]
-        if not sys.platform.startswith('linux'):    # 无桌面的Linux下无法输出，如果是有桌面的Linux，如Ubuntu，可去掉这一行
-            for i in range(label_column_num):
-                plt.figure(i+1)                     # 预测数据绘制
-                plt.plot(X, label_data[:, i], label='label', color='b')
-                plt.plot(X, predict_data[:, i], label='predict', color='g')
-                # plt.plot(predict_X, dq_data[:, i], label='dq', color='y')
-                # plt.title("Predict actual {} power with {}".format(label_name[i], self.opt.used_frame))
-                self.logger.info("The predicted power {} for the last {} point(s) is: ".format(label_name[i], self.opt.predict_points) +
-                      str(np.squeeze(predict_data[-self.opt.predict_points:, i])))
-
-                if self.opt.do_figure_save:
-                    plt.savefig(self.opt.figure_save_path+"{}predict_{}_with_{}.png".format(self.opt.continue_flag, label_name[i], self.opt.used_frame))
-
-            plt.show()
-
-    def tangle_results(self):
-        pass

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/config.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/3/17 14:46
-# file: config.py
-# author: David
-# company: shenyang JY
-
-import yaml
-import argparse
-
-
-class myargparse(argparse.ArgumentParser):
-    def __init__(self, discription, add_help):
-        super(myargparse, self).__init__(description=discription, add_help=add_help)
-        # default_config_parser = parser = argparse.ArgumentParser(
-        #     description='Training Config', add_help=False)
-        self.add_argument(
-            '-c',
-            '--config_yaml',
-            default=
-            'config.yml',
-            type=str,
-            metavar='FILE',
-            help='YAML config file specifying default arguments')
-
-        feature_columns = ['C_TIME', 'C_REAL_VALUE', 'C_FP_VALUE']
-        # feature_columns = list(range(1, 28))
-        label_columns = ['C_REAL_VALUE']
-
-        label_in_feature_index = (lambda x, y: [x.index(i) for i in y])(feature_columns, label_columns)  # 因为feature不一定从0开始
-
-        # 在控制台可以指定的参数， yml中没有
-        self.add_argument('--feature_columns', type=list, default=feature_columns, help='要作为特征的列')
-
-        self.add_argument('--label_columns', type=list, default=label_columns, help='要预测的列')
-
-        self.add_argument('--label_in_feature_index', type=list, default=label_in_feature_index, help='标签在特征列的索引')
-
-        self.add_argument('--input_size', type=int, default=len(feature_columns), help='输入维度')
-
-        self.add_argument('--output_size', type=int, default=len(label_columns), help='输出维度')
-
-        self.add_argument("--train_data_path", type=str, default=None,help='数据集地址')  # train_data_path yml中有
-
-        # model_name 和 model_save_path 这两个参数根据yml中的参数拼接而成
-
-        self.add_argument('--model_name', type=str, default=None, help='模型名称')
-
-        self.add_argument('--model_save_path', type=str, default=None, help='模型保存地址')
-
-
-    def _init_dir(self, opt):
-        import os, time
-        # 在这里给opt赋值
-        opt.model_name = "model_" + opt.continue_flag + opt.used_frame + opt.model_postfix[opt.used_frame]
-        opt.model_save_path = './checkpoint/' + opt.model_name + "/"
-        if not os.path.exists(opt.model_save_path):
-            os.makedirs(opt.model_save_path)    # makedirs 递归创建目录
-        if not os.path.exists(opt.figure_save_path):
-            os.mkdir(opt.figure_save_path)
-        if opt.do_train and (opt.do_log_save_to_file or opt.do_train_visualized):
-            cur_time = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
-            log_save_path = opt.log_save_path + cur_time + '_' + opt.used_frame + "/"
-            os.makedirs(log_save_path)
-
-
-# YAML should override the argparser's content
-    def _parse_args_and_yaml(self):
-        given_configs, remaining = self.parse_known_args()
-        if given_configs.config_yaml:
-            with open(given_configs.config_yaml, 'r', encoding='utf-8') as f:
-                cfg = yaml.safe_load(f)
-                self.set_defaults(**cfg)
-
-        # The main arg parser parses the rest of the args, the usual
-        # defaults will have been overridden if config file specified.
-        opt = self.parse_args(remaining)
-        self._init_dir(opt)
-        # Cache the args as a text string to save them in the output dir later
-        opt_text = yaml.safe_dump(opt.__dict__, default_flow_style=False)
-        return opt, opt_text
-
-
-    def parse_args_and_yaml(self):
-        return self._parse_args_and_yaml()[0]
-
-
-if __name__ == "__main__":
-    # opt = _parse_args_and_yaml()
-    pass
-
-
-
-

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/config.yml

@@ -1,78 +0,0 @@
-Model:
-  batch_size: 64
-  dropout_rate: 0.2
-  epoch: 20
-  hidden_size: 128
-  learning_rate: 0.001
-  lstm_layers: 2
-  patience: 5
-  random_seed: 42
-  time_step: 16
-add_train: false
-continue_flag: ''
-data_format:
-  dq: dq.csv
-  envir: "\u73AF\u5883\u6570\u636E.csv"
-  nwp: nwp.csv
-  rp: rp.csv
-  formula: t_forecast_power_ultra_short_term_his.csv
-debug_model: false
-debug_num: 500
-do_continue_train: false
-do_figure_save: false
-do_log_print_to_screen: true
-do_log_save_to_file: true
-do_predict: true
-do_train: false
-do_train_visualized: true
-excel_data_path: ./data/J00307/
-figure_save_path: ./figure/
-is_continuous_predict: True
-log_save_path: ./log/
-mean:
-  C_AIRT: 10.305992230762874
-  C_CELLT: 10.664897925448384
-  C_DIFFUSER: 143.2639061079428
-  C_DIFFUSERDA: 6.571077155136789
-  C_DIRECTR: 68.21328208942887
-  C_DIRECTRDA: 3.163283039920654
-  C_FORECAST: 3.1419734966774113
-  C_GLOBALR: 173.2587817174973
-  C_GLOBALRDA: 7.756491280271097
-  C_HOURDA: 1.998222150590958
-  C_P: 947.7830440532276
-  C_RH: 55.59672286965865
-  C_VALUE: 3.404744648318043
-  C_WD: 212.88300686007108
-  C_WS: 1.802446483180428
-model_postfix:
-  keras: .h5
-  pytorch: .pth
-  tensorflow: .ckpt
-predict_points: 16
-shuffle_train_data: false
-std:
-  C_AIRT: 12.127220611319888
-  C_CELLT: 12.654848145970181
-  C_DIFFUSER: 230.93680419867772
-  C_DIFFUSERDA: 6.4933162833681415
-  C_DIRECTR: 166.61348332191056
-  C_DIRECTRDA: 4.991297839913351
-  C_FORECAST: 4.447082956749344
-  C_GLOBALR: 258.87947949591955
-  C_GLOBALRDA: 7.9174382136573955
-  C_HOURDA: 2.9110230573747247
-  C_P: 25.75152505719027
-  C_RH: 22.445059526990818
-  C_VALUE: 5.013868885103326
-  C_WD: 112.90029001408325
-  C_WS: 1.6575249140627502
-train_data_path: ./data/
-train_data_rate: 0.9
-use_cuda: false
-used_frame: keras
-valid_data_rate: 0.15
-
-is_photovoltaic: True
-cap: 110
-envir_columns: 16

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/data_analyse.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/4/12 18:57
-# file: data_analyse.py
-# author: David
-# company: shenyang JY
-import sys
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-from data_utils import *
-
-class data_analyse(object):
-    def __init__(self, opt, logger, process):
-        self.opt = opt
-        self.logger = logger
-        self.ds = process
-
-    def formula_acc(self):
-        excel_data_path = self.opt.excel_data_path
-        data_format = self.opt.data_format
-        formula_path = excel_data_path + data_format["formula"]
-        formula = pd.read_csv(formula_path, usecols=['C_ABLE_VALUE', 'C_FORECAST_HOW_LONG_AGO', 'C_FORECAST_TIME'])
-        formula["C_FORECAST_TIME"] = formula["C_FORECAST_TIME"].apply(timestr_to_datetime)
-        formula = formula.rename(columns={"C_FORECAST_TIME": "C_TIME"})
-        formula = formula.loc[formula['C_FORECAST_HOW_LONG_AGO'] == 16]
-        return formula
-
-    def calculate_acc(self, label_data, predict_data):
-        loss = np.sum((label_data - predict_data) ** 2) / len(label_data)  # mse
-        loss_sqrt = np.sqrt(loss)  # rmse
-        loss_acc = 1 - loss_sqrt / self.opt.cap
-        return loss_acc
-
-    def get_16_points(self, results):
-        # results为模型预测的一维数组，遍历，取每16个点的最后一个点
-        preds = []
-        for res in results:
-            preds.append(res.iloc[-1].values)
-        return np.array(preds)
-
-    def predict_acc(self, predict_data, dfy):
-        predict_data = predict_data * self.ds.std['C_REAL_VALUE'] + self.ds.mean['C_REAL_VALUE']
-        dfs = dfy[0]
-        for i in range(1, len(dfy)):
-            dfs.extend(dfy[i])
-        for i, df in enumerate(dfs):
-            df["PREDICT"] = predict_data[i]
-            dfs[i] = df
-        data = self.get_16_points(dfs)
-        df = pd.DataFrame(data, columns=['C_TIME', 'C_REAL_VALUE', 'C_FP_VALUE', 'PREDICT'])
-        # label_data = label_data.reshape((-1, self.opt.output_size))
-        # label_data 要进行反归一化
-        df.to_csv(self.opt.excel_data_path + "dq+rp.csv")
-        formula = self.formula_acc()
-        df = pd.merge(df, formula, on='C_TIME')
-
-        label_name = [self.opt.feature_columns[i] for i in self.opt.label_in_feature_index]
-        loss_norm = self.calculate_acc(df['C_REAL_VALUE'], df['PREDICT'])
-
-        self.logger.info("The mean squared error of power {} is ".format(label_name) + str(loss_norm))
-
-        loss_norm = self.calculate_acc(df['C_REAL_VALUE'], df['C_FP_VALUE'])
-
-        self.logger.info("The mean squared error of power {} is ".format(label_name) + str(loss_norm))
-
-        loss_norm = self.calculate_acc(df['C_REAL_VALUE'], df['C_ABLE_VALUE'])
-
-        self.logger.info("The mean squared error of power {} is ".format(label_name) + str(loss_norm))
-        self.preidct_draw(df['C_REAL_VALUE'].values, df['PREDICT'].values)
-
-    def preidct_draw(self, label_data, predict_data):
-        X = list(range(label_data.shape[0]))
-        print("label_x = ", X)
-        label_column_num = len(self.opt.label_columns)
-        label_name = [self.opt.feature_columns[i] for i in self.opt.label_in_feature_index]
-        if not sys.platform.startswith('linux'):    # 无桌面的Linux下无法输出，如果是有桌面的Linux，如Ubuntu，可去掉这一行
-            for i in range(label_column_num):
-                plt.figure(i+1)                     # 预测数据绘制
-                plt.plot(X, label_data, label='label', color='b')
-                plt.plot(X, predict_data, label='predict', color='g')
-                # plt.plot(predict_X, dq_data[:, i], label='dq', color='y')
-                # plt.title("Predict actual {} power with {}".format(label_name[i], self.opt.used_frame))
-                # self.logger.info("The predicted power {} for the last {} point(s) is: ".format(label_name[i], self.opt.predict_points) +
-                #       str(np.squeeze(predict_data[-self.opt.predict_points:, i])))
-
-                if self.opt.do_figure_save:
-                    plt.savefig(self.opt.figure_save_path+"{}predict_{}_with_{}.png".format(self.opt.continue_flag, label_name[i], self.opt.used_frame))
-
-            plt.show()
-
-    def tangle_results(self):
-        pass

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/data_features.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/4/12 17:42
-# file: data_features.py
-# author: David
-# company: shenyang JY
-import pandas as pd
-from sklearn.model_selection import train_test_split
-import numpy as np
-from data_utils import *
-
-
-class data_features(object):
-    def __init__(self, opt, mean, std):
-        self.opt = opt
-        self.time_step = self.opt.Model["time_step"]
-        self.mean = mean
-        self.std = std
-        self.columns = list()
-
-    def get_train_data(self, dfs):
-        train_x, valid_x, train_y, valid_y = [], [], [], []
-        for df in dfs:
-            datax, datay = self.get_data_features(df)
-            trainx = np.array(datax)
-            trainy = [y['C_REAL_VALUE'].values for y in datay]
-            trainy = np.expand_dims(np.array(trainy), axis=-1)  # 在最后一维加一维度
-            tx, vx, ty, vy = train_test_split(trainx, trainy, test_size=self.opt.valid_data_rate,
-                                                                  random_state=self.opt.Model["random_seed"],
-                                                                  shuffle=self.opt.shuffle_train_data)  # 划分训练和验证集
-            train_x.append(tx)
-            valid_x.append(vx)
-            train_y.append(ty)
-            valid_y.append(vy)
-
-        train_x = np.concatenate(train_x, axis=0)
-        valid_x = np.concatenate(valid_x, axis=0)
-        train_y = np.concatenate(train_y, axis=0)
-        valid_y = np.concatenate(valid_y, axis=0)
-
-        train_x = self.norm_features(train_x)
-        valid_x = self.norm_features(valid_x)
-        train_y = self.norm_label(train_y)
-        valid_y = self.norm_label(valid_y)
-
-        return train_x, valid_x, train_y, valid_y
-
-    def get_test_data(self, dfs):
-        test_x, test_y, data_y = [], [], []
-        for df in dfs:
-            datax, datay = self.get_data_features(df)
-            trainx = np.array(datax)
-            trainy = [y['C_REAL_VALUE'].values for y in datay]
-            trainy = np.expand_dims(np.array(trainy), axis=-1)  # 在最后一维加一维度
-            test_x.append(trainx)
-            test_y.append(trainy)
-            data_y.append(datay)
-
-        test_x = np.concatenate(test_x, axis=0)
-        test_y = np.concatenate(test_y, axis=0)
-
-        test_x = self.norm_features(test_x)
-        test_y = self.norm_label(test_y)
-
-        return test_x, test_y, data_y
-
-    def get_data_features(self, df):
-        norm_data = df.reset_index()
-        feature_data = norm_data[:-self.opt.predict_points]
-        label_data = norm_data[self.opt.predict_points:].reset_index(drop=True)
-        time_step = self.opt.Model["time_step"]
-        time_step_loc = time_step - 1
-        train_num = int(len(feature_data))
-        time_rp = [feature_data.loc[i:i + time_step_loc, ['C_TIME', 'C_REAL_VALUE']] for i in range(train_num - time_step)]
-        dq = [label_data.loc[i:i + time_step_loc, 'C_FP_VALUE'] for i in range(train_num - time_step)]
-        features_x, features_y = [], []
-        for row in zip(time_rp, dq):
-            row0 = row[0]
-            row1 = row[1]
-            row0['C_FP_VALUE'] = row1
-            row0.set_index('C_TIME', inplace=True, drop=False)
-            row0["C_TIME"] = row0["C_TIME"].apply(datetime_to_timestr)
-            features_x.append(row0)
-        self.columns = row0.columns.tolist()
-
-        features_y = [label_data.loc[i:i + time_step_loc, ['C_TIME', 'C_REAL_VALUE', 'C_FP_VALUE']] for i in range(train_num - time_step)]
-        return features_x, features_y
-
-    def norm_features(self, data: np.ndarray):
-        mean = np.array([self.mean[col] for col in self.columns])
-        std = np.array([self.std[col] for col in self.columns])
-        data = (data - mean) / std  # 归一化
-        return data
-
-    def norm_label(self, label_data: np.ndarray):
-        return (label_data - self.mean['C_REAL_VALUE']) / self.std['C_REAL_VALUE']

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/data_process.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/3/17 10:10
-# file: main.py
-# author: David
-# company: shenyang JY
-import pandas as pd
-import numpy as np
-from data_utils import *
-import yaml
-
-
-class data_process(object):
-    def __init__(self, opt):
-        self.std = None
-        self.mean = None
-        self.opt = opt
-        # 都是在ndarray量纲下进行计算
-        # self.norm_data = (self.tables[:, 1:] - self.mean) / self.std  # 归一化，去量纲
-        # self.norm_data.insert(0, 'C_TIME', self.tables['C_TIME'])
-        # self.set_yml({'mean': self.mean.to_dict(), 'std': self.std.to_dict()})
-        # self.start_num_in_test = 0
-
-    def get_processed_data(self):
-        excel_data_path = self.opt.excel_data_path
-        data_format = self.opt.data_format
-        dq_path = excel_data_path + data_format["dq"]
-        rp_path = excel_data_path + data_format["rp"]
-        envir_path = excel_data_path + data_format["envir"]
-
-        dq_columns = ['C_FORECAST_TIME', 'C_FP_VALUE']
-        rp_columns = ['C_TIME', 'C_REAL_VALUE']  # 待优化 ["'C_TIME'", "'C_REAL_VALUE'"] 原因：csv 字符串是单引号''，read_csv带单引号
-
-        # envir = self.read_data(envir_path).loc[:, "C_TIME":]  # 待优化 导出csv按照表的列顺序 read_csv按照csv列顺序读取
-        # envir = self.data_cleaning(envir)
-        # envir["C_TIME"] = envir["C_TIME"].apply(timestr_to_datetime)
-        # envir.set_index('C_TIME', inplace=True)
-        # envir = self.drop_duplicated(envir)
-
-        # 读取的df，经过时序转换、清洗、去重三部曲
-        dq = self.read_data(dq_path, dq_columns)
-        dq = dq.rename(columns={"C_FORECAST_TIME": "C_TIME"})
-        dq["C_TIME"] = dq["C_TIME"].apply(timestr_to_datetime)
-        dq.set_index('C_TIME', inplace=True)
-        dq = self.data_cleaning(dq)
-        dq = self.drop_duplicated(dq)
-
-        rp = self.read_data(rp_path, rp_columns)
-        rp["C_TIME"] = rp["C_TIME"].apply(timestr_to_datetime)
-        rp.set_index('C_TIME', inplace=True)  # nan也可以设置索引列
-        rp = self.data_cleaning(rp)
-        rp = self.drop_duplicated(rp)
-
-        df = self.tables_unite(rp, dq)
-        dfs = self.missing_time_splite(df)
-        dfs = [self.data_fill(df) for df in dfs]
-        self.norm(dfs)  # 归一化 待解决
-        return dfs
-
-    def norm(self, dfs):
-        df = pd.concat(dfs, axis=0)
-        df = df.reset_index()
-        df["C_TIME"] = df["C_TIME"].apply(datetime_to_timestr)
-        mean = np.mean(df, axis=0)  # 数据的均值
-        std = np.std(df, axis=0)  # 标准差
-        if hasattr(self.opt, 'mean') is False or hasattr(self.opt, 'std') is False:
-            self.set_yml({'mean': mean.to_dict(), 'std': std.to_dict()})
-        self.mean, self.std = mean.to_dict(), std.to_dict()
-
-    def data_cleaning(self, data):
-        data = data.replace(-99, np.nan)
-        # nan 超过30% 删除
-        data = data.dropna(axis=1, thresh=len(data)*0.7)
-        # nan 替换成0 本周问题 1.卷积学习，0是否合适？
-        data = data.replace(np.nan, 0)
-        # 删除取值全部相同的列
-        data = data.loc[:, (data != data.iloc[0]).any()]
-        return data
-
-    def missing_time_splite(self, df):
-        dt = pd.Timedelta(minutes=15)
-        day1 = pd.Timedelta(days=1)
-        cnt = 0
-        cnt1 = 0
-        start_index = 0
-        dfs = []
-        for i in range(1, len(df)):
-            if df.index[i] - df.index[i-1] >= day1:
-                df_x = df.iloc[start_index:i, ]
-                dfs.append(df_x)
-                start_index = i
-                cnt1 += 1
-            if df.index[i] - df.index[i-1] != dt:
-                print(df.index[i-1], end=" ~ ")
-                print(df.index[i])
-                cnt += 1
-        dfs.append(df.iloc[start_index:, ])
-        print("数据总数：", len(df), "，缺失段数：", cnt, "其中，超过一天的段数：", cnt1)
-        return dfs
-
-    def data_fill(self, df):
-        df = df.resample('15T').bfill()
-        return df
-
-    def set_yml(self, yml_dict):
-        with open(self.opt.config_yaml, 'r', encoding='utf-8') as f:
-            cfg = yaml.safe_load(f)
-        for k, v in yml_dict.items():
-            cfg[k] = v
-        with open(self.opt.config_yaml, 'w') as f:
-            yaml.safe_dump(cfg, f, default_flow_style=False)
-
-    def read_data(self, path, cols=None, index_col=None):
-        init_data = pd.read_csv(path, usecols=cols, index_col=index_col)
-        return init_data
-
-    def filter_data(self):
-        check_table = self.tables[:, 2]  # 实际功率不能为0，为0代表没发电
-        preserve_index = list(np.nonzero(check_table)[0])
-        indexs = list(range(len(self.tables)))
-        del_index = list(set(indexs) - set(preserve_index))
-        self.tables = np.delete(self.tables, del_index, axis=0)
-        return self.tables
-
-    def drop_duplicated(self, df):
-        df = df.groupby(level=0).mean()  # DatetimeIndex时间索引去重
-        return df
-
-    def tables_unite(self, t1, t2):
-        return pd.merge(t1, t2, left_index=True, right_index=True)
-
-
-if __name__ == "__main__":
-    ds = DataSet()
-    # dq = ds.read_data(dq_path, dq_columns)[0]
-    # rp = ds.read_data(rp_path, rp_columns)[0]
-    # # rp_average(rp)    # 计算平均功率
-    # envir = ds.read_data(envir_path, envir_columns)[0]
-    # tables = ds.tables_integra(dq, rp, envir)
-    # ds.tables_norm_result(tables)

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/data_utils.py

@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/3/22 17:17
-# file: dpdUtils.py
-# author: David
-# company: shenyang JY
-
-
-import time, datetime
-
-
-class ValidationError(Exception):
-    def __init__(self, message):
-        self.message = message
-
-
-def timestamp_to_datetime(ts):
-    if type(ts) is not int:
-        raise ValueError("timestamp-时间格式必须是整型")
-    if len(str(ts)) == 13:
-        return datetime.datetime.fromtimestamp(ts/1000)
-    elif len(str(ts)) == 10:
-        return datetime.datetime.fromtimestamp(ts)
-    else:
-        raise ValueError("timestamp-时间格式长度错误")
-
-
-def datetime_to_timestamp(dt, len):
-    if len not in (10, 13):
-        raise ValueError("timestamp-时间戳转换长度错误")
-    if len == 10:
-        return int(round(time.mktime(dt.timetuple())))
-    else:
-        return int(round(time.mktime(dt.timetuple()))*1000)
-
-
-def datetime_to_timestr(dt):
-    return int(dt.strftime('%m%d%H%M'))
-
-
-def timestr_to_datetime(time_data):
-    """
-    将时间戳或时间字符串转换为datetime.datetime类型
-    :param time_data: int or str
-    :return:datetime.datetime
-    """
-    if isinstance(time_data, float):
-        result = timestamp_to_datetime(int(time_data))
-    elif isinstance(time_data, int):
-        result = timestamp_to_datetime(time_data)
-    elif isinstance(time_data, str):
-        if len(time_data) == 10:
-            result = datetime.datetime.strptime(time_data, '%d/%m/%Y')
-            # result = datetime.datetime.strptime(time_data, '%Y-%m-%d')
-        elif len(time_data) in {17, 18, 19}:
-            result = datetime.datetime.strptime(time_data, '%d/%m/%Y %H:%M:%S')   # strptime字符串解析必须严格按照字符串中的格式
-            # result = datetime.datetime.strptime(time_data, '%Y-%m-%d %H:%M:%S')
-        else:
-            raise ValidationError("时间字符串长度不满足要求！")
-    return result
-
-
-def timestamp_to_timestr(t):
-    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(t))
-

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/logger.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/3/20 15:19
-# file: logger.py
-# author: David
-# company: shenyang JY
-
-import logging, sys
-from logging.handlers import RotatingFileHandler
-
-
-def load_logger(config):
-    logger = logging.getLogger()
-    logger.setLevel(level=logging.DEBUG)
-
-    # StreamHandler
-    if config.do_log_print_to_screen:
-        stream_handler = logging.StreamHandler(sys.stdout)
-        stream_handler.setLevel(level=logging.INFO)
-        formatter = logging.Formatter(datefmt='%Y/%m/%d %H:%M:%S',
-                                      fmt='[ %(asctime)s ] %(message)s')
-        stream_handler.setFormatter(formatter)
-        logger.addHandler(stream_handler)
-
-    # FileHandler
-    if config.do_log_save_to_file:
-        file_handler = RotatingFileHandler(config.log_save_path + "out.log", maxBytes=1024000, backupCount=5)
-        file_handler.setLevel(level=logging.INFO)
-        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        file_handler.setFormatter(formatter)
-        logger.addHandler(file_handler)
-
-        # 把config信息也记录到log 文件中
-        config_dict = {}
-        for key in dir(config):
-            if not key.startswith("_"):
-                config_dict[key] = getattr(config, key)
-        config_str = str(config_dict)
-        config_list = config_str[1:-1].split(", '")
-        config_save_str = "\nConfig:\n" + "\n'".join(config_list)
-        logger.info(config_save_str)
-
-    return logger

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/model/model_keras.py

@@ -1,48 +0,0 @@
-# -*- coding: UTF-8 -*-
-from keras.layers import Input, Dense, LSTM
-from keras.models import Model
-from keras.callbacks import ModelCheckpoint, EarlyStopping
-
-
-def get_keras_model(opt):
-    input1 = Input(shape=(opt.Model['time_step'], opt.input_size))
-    lstm = input1
-    for i in range(opt.Model['lstm_layers']):
-        lstm = LSTM(units=opt.Model['hidden_size'],dropout=opt.Model['dropout_rate'],return_sequences=True)(lstm)
-    output = Dense(opt.output_size)(lstm)
-    model = Model(input1, output)
-    model.compile(loss='mse', optimizer='adam')     # metrics=["mae"]
-    return model
-
-
-def gpu_train_init():
-    import tensorflow as tf
-    from keras.backend.tensorflow_backend import set_session
-    sess_config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
-    sess_config.gpu_options.per_process_gpu_memory_fraction = 0.7  # 最多使用70%GPU内存
-    sess_config.gpu_options.allow_growth=True   # 初始化时不全部占满GPU显存, 按需分配
-    sess = tf.Session(config=sess_config)
-    set_session(sess)
-
-
-def train(opt, train_and_valid_data):
-    if opt.use_cuda: gpu_train_init()
-    train_X, train_Y, valid_X, valid_Y = train_and_valid_data
-    model = get_keras_model(opt)
-    model.summary()
-    if opt.add_train:
-        model.load_weights(opt.model_save_path + opt.model_name)
-
-    check_point = ModelCheckpoint(filepath=opt.model_save_path + opt.model_name, monitor='val_loss',
-                                    save_best_only=True, mode='auto')
-    early_stop = EarlyStopping(monitor='val_loss', patience=opt.Model['patience'], mode='auto')
-    model.fit(train_X, train_Y, batch_size=opt.Model['batch_size'], epochs=opt.Model['epoch'], verbose=2,
-              validation_data=(valid_X, valid_Y), callbacks=[check_point, early_stop])
-
-
-def predict(config, test_X):
-    model = get_keras_model(config)
-    model.load_weights(config.model_save_path + config.model_name)
-    result = model.predict(test_X, batch_size=1)
-    # result = result.reshape((-1, config.output_size))
-    return result

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/requirements.txt

@@ -1,8 +0,0 @@
-sklearn
-pandas
-argparse
-keras
-tensorflow==1.15
-matplotlib>=3.0.2
-numpy>=1.14.6
-scipy>=1.1.0

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/run_case1.py

@@ -1,55 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-import numpy as np
-import os
-from data_process import data_process
-from data_features import data_features
-from logger import load_logger
-from config import myargparse
-from data_analyse import data_analyse
-frame = "keras"
-
-if frame == "keras":
-    from model.model_keras import train, predict
-    os.environ["TF_CPP_MIN_LOG_LEVEL"] = '3'
-else:
-    raise Exception("Wrong frame seletion")
-
-
-def main():
-    parse = myargparse(discription="training config", add_help=False)
-    opt = parse.parse_args_and_yaml()
-    logger = load_logger(opt)
-    try:
-        np.random.seed(opt.Model["random_seed"])
-        process = data_process(opt=opt)
-        dfs = process.get_processed_data()
-        features = data_features(opt=opt, mean=process.mean, std=process.std)
-        if opt.do_train:
-            train_X, valid_X, train_Y, valid_Y = features.get_train_data([dfs[0][:'2021/8/1'], dfs[1][:'2022/3/1']])
-            train(opt, [train_X, train_Y, valid_X, valid_Y])
-        if opt.do_predict:
-            test_X, test_Y, df_Y = features.get_test_data([dfs[0]['2021/8/1':'2021/9/6'], dfs[1]['2022/3/1':'2022/4/4']])
-            result = predict(opt, test_X)       # 这里输出的是未还原的归一化预测数据
-            analyse = data_analyse(opt, logger, process)
-            analyse.predict_acc(result, df_Y)
-    except Exception:
-        logger.error("Run Error", exc_info=True)
-
-
-if __name__ == "__main__":
-    import argparse
-    # argparse方便于命令行下输入参数，可以根据需要增加更多
-    # parser = argparse.ArgumentParser()
-    # parser.add_argument("-t", "--do_train", default=False, type=bool, help="whether to train")
-    # parser.add_argument("-p", "--do_predict", default=True, type=bool, help="whether to train")
-    # parser.add_argument("-b", "--batch_size", default=64, type=int, help="batch size")
-    # parser.add_argument("-e", "--epoch", default=20, type=int, help="epochs num")
-    # args = parser.parse_args()
-
-    # con = Config()
-    # for key in dir(args):               # dir(args) 函数获得args所有的属性
-    #     if not key.startswith("_"):     # 去掉 args 自带属性，比如__name__等
-    #         setattr(con, key, getattr(args, key))   # 将属性值赋给Config
-    main()
-

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/run_case_history.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/3/20 9:23
-# file: run_case_history.py
-# author: David
-# company: shenyang JY
-
-class Data:
-    def __init__(self, config):
-        self.config = config
-        self.data, self.data_column_name = self.read_data()
-
-        self.data_num = self.data.shape[0]
-        self.train_num = int(self.data_num * self.config.train_data_rate)
-
-        self.mean = np.mean(self.data, axis=0)              # 数据的均值和方差
-        self.std = np.std(self.data, axis=0)
-        self.norm_data = (self.data - self.mean)/self.std   # 归一化，去量纲
-
-        self.start_num_in_test = 0      # 测试集中前几天的数据会被删掉，因为它不够一个time_step
-
-    def read_data(self):                # 读取初始数据
-        if self.config.debug_mode:
-            init_data = pd.read_csv(self.config.train_data_path, nrows=self.config.debug_num,
-                                    usecols=self.config.feature_columns)
-        else:
-            init_data = pd.read_csv(self.config.train_data_path, usecols=self.config.feature_columns)
-        init_data = self.filter_data(init_data)
-        return init_data.values, init_data.columns.tolist()     # .columns.tolist() 是获取列名
-
-    def filter_data(self, init_data):
-        return init_data[init_data.apply(np.sum, axis=1)!=0]
-
-    def get_train_and_valid_data(self):
-        feature_data = self.norm_data[:self.train_num]
-        label_data = self.norm_data[: self.train_num,
-                                    self.config.label_in_feature_index]    # 将延后几天的数据作为label
-
-        if not self.config.do_continue_train:
-            # 在非连续训练模式下，每time_step行数据会作为一个样本，两个样本错开一行，比如：1-20行，2-21行。。。。
-            train_x, train_y = [], []
-            for i in range(self.train_num-self.config.time_step*2):
-                p1 = feature_data[:, 0][i:i+self.config.start_predict_point]
-                p2 = feature_data[:, 1][i+self.config.start_predict_point:i+self.config.start_predict_point*2]
-                p = [list(t) for t in zip(p1, p2)]  # 实际功率， 预测功率 是一组特征值
-                l = label_data[i+self.config.start_predict_point:i+self.config.start_predict_point*2]
-                train_x.append(p)
-                train_y.append(l)
-            # train_x = [feature_data[i:i+self.config.time_step] for i in range(self.train_num-self.config.time_step)]
-            # train_y = [label_data[i+self.config.start_predict_point:i+self.config.time_step] for i in range(self.train_num-self.config.time_step)]
-            # 这里选取后16个点 作为 预测及
-        else:
-            # 在连续训练模式下，每time_step行数据会作为一个样本，两个样本错开time_step行，
-            # 比如：1-20行，21-40行。。。到数据末尾，然后又是 2-21行，22-41行。。。到数据末尾，……
-            # 这样才可以把上一个样本的final_state作为下一个样本的init_state，而且不能shuffle
-            # 目前本项目中仅能在pytorch的RNN系列模型中用
-            train_x = [feature_data[start_index + i*self.config.time_step : start_index + (i+1)*self.config.time_step]
-                       for start_index in range(self.config.time_step)
-                       for i in range((self.train_num - start_index) // self.config.time_step)]
-            train_y = [label_data[start_index + i*self.config.time_step : start_index + (i+1)*self.config.time_step]
-                       for start_index in range(self.config.time_step)
-                       for i in range((self.train_num - start_index) // self.config.time_step)]
-
-        train_x, train_y = np.array(train_x), np.array(train_y)
-
-        train_x, valid_x, train_y, valid_y = train_test_split(train_x, train_y, test_size=self.config.valid_data_rate,
-                                                              random_state=self.config.random_seed,
-                                                              shuffle=self.config.shuffle_train_data)   # 划分训练和验证集，并打乱
-        return train_x, valid_x, train_y, valid_y
-
-
-class Config:
-    # 数据参数
-    # feature_columns = list(range(2, 9))     # 要作为feature的列，按原数据从0开始计算，也可以用list 如 [2,4,6,8] 设置
-    feature_columns = list(range(1, 3))
-    # label_columns = [4, 5]                  # 要预测的列，按原数据从0开始计算, 如同时预测第四，五列 最低价和最高价
-    label_columns = [1]
-    # label_in_feature_index = [feature_columns.index(i) for i in label_columns]  # 这样写不行
-    label_in_feature_index = (lambda x,y: [x.index(i) for i in y])(feature_columns, label_columns)  # 因为feature不一定从0开始
-
-    predict_day = 1             # 预测未来几天
-    predict_points = 16
-    # 网络参数
-    input_size = len(feature_columns)
-    output_size = len(label_columns)
-
-    hidden_size = 128           # LSTM的隐藏层大小，也是输出大小
-    lstm_layers = 2             # LSTM的堆叠层数
-    dropout_rate = 0.2          # dropout概率
-    time_step = 16             # 这个参数很重要，是设置用前多少个点的数据来预测，也是LSTM的time step数，请保证训练数据量大于它
-    start_predict_point = 16
-
-    # 训练参数
-    do_train = True
-    do_predict = True
-    add_train = False           # 是否载入已有模型参数进行增量训练
-    shuffle_train_data = False   # 是否对训练数据做shuffle
-    use_cuda = False            # 是否使用GPU训练
-
-    train_data_rate = 0.95      # 训练数据占总体数据比例，测试数据就是 1-train_data_rate
-    valid_data_rate = 0.15      # 验证数据占训练数据比例，验证集在训练过程使用，为了做模型和参数选择
-
-    batch_size = 64
-    learning_rate = 0.001
-    epoch = 20                  # 整个训练集被训练多少遍，不考虑早停的前提下
-    patience = 5                # 训练多少epoch，验证集没提升就停掉
-    random_seed = 42            # 随机种子，保证可复现
-
-    do_continue_train = False    # 每次训练把上一次的final_state作为下一次的init_state，仅用于RNN类型模型，目前仅支持pytorch
-    continue_flag = ""           # 但实际效果不佳，可能原因：仅能以 batch_size = 1 训练
-    if do_continue_train:
-        shuffle_train_data = False
-        batch_size = 1
-        continue_flag = "continue_"
-
-    # 训练模式
-    debug_mode = False  # 调试模式下，是为了跑通代码，追求快
-    debug_num = 500  # 仅用debug_num条数据来调试
-
-    # 框架参数
-    used_frame = frame  # 选择的深度学习框架，不同的框架模型保存后缀不一样
-    model_postfix = {"pytorch": ".pth", "keras": ".h5", "tensorflow": ".ckpt"}
-    model_name = "model_" + continue_flag + used_frame + model_postfix[used_frame]
-
-    # 路径参数
-    train_data_path = "./data/J00285.csv"
-    model_save_path = "./checkpoint/" + used_frame + "/"
-    figure_save_path = "./figure/"
-    log_save_path = "./log/"
-    do_log_print_to_screen = True
-    do_log_save_to_file = True                  # 是否将config和训练过程记录到log
-    do_figure_save = False
-    do_train_visualized = False          # 训练loss可视化，pytorch用visdom，tf用tensorboardX，实际上可以通用, keras没有
-    if not os.path.exists(model_save_path):
-        os.makedirs(model_save_path)    # makedirs 递归创建目录
-    if not os.path.exists(figure_save_path):
-        os.mkdir(figure_save_path)
-    if do_train and (do_log_save_to_file or do_train_visualized):
-        cur_time = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
-        log_save_path = log_save_path + cur_time + '_' + used_frame + "/"
-        os.makedirs(log_save_path)
-

ipfcst-forecast-wind-LSTM-v2.0-dq+rp/test.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# time: 2023/4/11 15:58
-# file: test.py
-# author: David
-# company: shenyang JY
-import pandas as pd
-import numpy as np
-# index = pd.date_range('1/1/2000', periods=9, freq='T')
-# series = pd.Series(range(9), index=index)
-# df = pd.DataFrame({'value': series})
-# series1 = series.resample('3T').sum()
-# series2 = series.resample('3T', label='right').sum()
-# series3 = series.resample('3T', label='right', closed='right').sum()
-# series4 = series.resample('30S').asfreq()
-# series5 = series.resample('30S').bfill()
-# print(series)
-# print(series1)
-# print(series2)
-# print(series3)
-# print(series4)
-# print("---", series5)
-
-# x = np.random.randint(1,100,20).reshape((10,2))
-# print(x)
-# from sklearn.model_selection import train_test_split
-#
-# x_train, x_test = train_test_split(x, test_size=0.2, random_state=1, shuffle=False)
-# print("x_train", x_train)
-# print("x_test", x_test)
-
-
-import numpy as np
-import pandas as pd
-#创建一组数据
-data = {'name': ['John', 'Mike', 'Mozla', 'Rose', 'David', 'Marry', 'Wansi', 'Sidy', 'Jack', 'Alic'],
-        'age': [20, 32, 29, np.nan, 15, 28, 21, 30, 37, 25],
-        'gender': [0, 0, 1, 1, 0, 1, 0, 0, 1, 1],
-        'isMarried': ['yes', 'yes', 'no', 'yes', 'no', 'no', 'no', 'yes', 'no', 'no']}
-label = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']
-df = pd.DataFrame(data, index=label)
-print(df.loc[:,'name'])
-pass