cluster_power完整代码

cluster_power.py

@@ -6,10 +6,47 @@
 # company: shenyang JY
 
 import os
+import re
 import numpy as np
 import pandas as pd
-# 从confinuous_data中拿数据
-cluster = []
+
+
+def read_cfs(cfs, input_path, output_path, is_folder=False):
+    if not os.path.exists(output_path):
+        os.makedirs(output_path)
+    dfs = {}
+    for j, ids in cfs.items():
+        if is_folder:
+            dirname = input_path.split('/')[-1]
+            x = re.findall('(?<=Continuous_Turbine_Data_).*?(?=_)',dirname)[0]
+            dfs_j = [pd.read_csv(os.path.join(input_path, f"turbine-{id}_{int(x)}.csv")) for id in ids]
+        else:
+            dfs_j = [pd.read_csv(os.path.join(input_path, f"turbine-{id}.csv")) for id in ids]
+        dfj, time_series = dfs_j[0].loc[:, ['C_TIME', 'C_WS', 'C_ACTIVE_POWER']], dfs_j[0]['C_TIME']
+        for df in dfs_j[1:]:
+            if df['C_TIME'].equals(time_series) is False:
+                print("风机之间的日期不一致！")
+                raise ValueError
+            dfj['C_ACTIVE_POWER'] += df['C_ACTIVE_POWER']
+            dfj['C_WS'] += df['C_WS']
+        dfj['C_WS'] /= len(dfj)
+        dfj.rename(columns=({'C_ACTIVE_POWER':'C_ACTIVE_POWER'+str(j), 'C_WS': 'C_WS'+str(j)}), inplace=True)
+        if is_folder:
+            dfj.to_csv(os.path.join(output_path, 'cluster_' + str(j) + '.csv'), index=False)
+        else:
+            dfj[20:].to_csv(os.path.join(output_path, 'cluster_' + str(j) + '.csv'), index=False)
+        dfs[j] = dfj
+    return dfs
+
+
+def get_cfs(cluster, turbine_id):
+    cfs = {}
+    for j in range(1, max(cluster) + 1):
+        arr_j = np.where(cluster == j)[0]  # cluster中聚类j的索引列表
+        cfs.setdefault(j, [turbine_id[k] for k in arr_j])
+    for key, value in cfs.items():
+        print("第{}组：{}".format(key, cfs[key]))
+    return cfs
 
 
 def cluster_power_list_file(cluster, turbine_id, input_path, output_path):
@@ -22,25 +59,11 @@ def cluster_power_list_file(cluster, turbine_id, input_path, output_path):
     """
     if not os.path.exists(output_path):
         os.makedirs(output_path)
-    files = os.listdir(input_path)
-    files.remove('turbine-144.csv')
-    assert len(cluster) == len(files)
-    dfs = [pd.read_csv(os.path.join(input_path, f)) for f in files]
-    cfs = {}
-    for j in range(1, max(cluster)+1):
-        arr_j = np.where(cluster == j)[0]   # cluster中聚类j的索引列表
-        cfs.setdefault(j, [turbine_id[k] for k in arr_j])
-        dfs_j = [dfs[k] for k in arr_j]   # cluster
-        dfj, time_series = dfs_j[0].loc[:, ['C_TIME','C_ACTIVE_POWER']], dfs_j[0]['C_TIME']
-        for df in dfs_j[1:]:
-            if df['C_TIME'].equals(time_series) is False:
-                print("风机之间的日期不一致！")
-                raise ValueError
-            dfj['C_ACTIVE_POWER'] += df['C_ACTIVE_POWER']
-        dfj.to_csv(os.path.join(output_path, 'power_' + str(j) + '.csv'), index=False)
-        paint_cluster_power(output_path)
-        for key, value in cfs.items():
-            print("第{}组：{}".format(key, cfs[key]))
+
+    cfs = get_cfs(cluster, turbine_id)
+    dfs = read_cfs(cfs, input_path, output_path)
+    dfs_cluster = pd.concat([df.set_index("C_TIME") for df in dfs.values()], join='inner', axis=1)
+    dfs_cluster.reset_index().to_csv(os.path.join(output_path, 'cluster_data.csv'), index=False)
 
 
 def cluster_power_list_folder(cluster, turbine_id, input_path, output_path):
@@ -53,42 +76,23 @@ def cluster_power_list_folder(cluster, turbine_id, input_path, output_path):
     """
     if not os.path.exists(output_path):
         os.makedirs(output_path)
-    assert len(cluster) == len(turbine_id)
-    continuous_data_path = input_path
-    # 遍历整个continuous_data_path文件夹
-    continuous_list = []
-    for dirpath, dirnames, filenames in os.walk(continuous_data_path):
-        dirname = dirpath.split('/')[-1]
-        filenames = [file for file in filenames if not file.startswith('turbine-144')]
-        x = [os.path.join(dirpath, filename) for filename in filenames]
-        continuous_list.append((x, dirname))
-    continuous_list.pop(0)
-    for i in range(len(continuous_list)):
-        cfs = {}
-        dfs = [pd.read_csv(path) for path in continuous_list[i][0]]
-        for j in range(1, max(cluster)+1):
-            arr_j = np.where(cluster == j)[0]   # cluster中聚类j的索引列表
-            cfs.setdefault(j, [turbine_id[k] for k in arr_j])
-            dfs_j = [dfs[k] for k in arr_j]   # cluster
-            dfj, time_series = dfs_j[0].loc[:, ['C_TIME','C_ACTIVE_POWER']], dfs_j[0]['C_TIME']
-            for df in dfs_j[1:]:
-                if df['C_TIME'].equals(time_series) is False:
-                    print("风机之间的日期不一致！")
-                    raise ValueError
-                dfj['C_ACTIVE_POWER'] += df['C_ACTIVE_POWER']
-            output_1 = os.path.join(output_path,  continuous_list[i][1])
-            if not os.path.exists(output_1):
-                os.makedirs(output_1)
-            dfj.to_csv(os.path.join(output_1, 'power_' + str(j) + '.csv'), index=False)
-        paint_cluster_power(output_1)
-        print("------{}的分组信息----".format(continuous_list[i][1]))
-        for key, value in cfs.items():
-            print("第{}组：{}".format(key, cfs[key]))
-
-def paint_cluster_power(cluster_path):
-    dfs = [pd.read_csv(os.path.join(cluster_path, file_path)).rename(columns={'C_ACTIVE_POWER':file_path.split('/')[-1][:-4]}) for file_path in os.listdir(cluster_path)]
+    continuous_list = [os.path.join(input_path, path) for path in os.listdir(input_path)]
+    cfs = get_cfs(cluster, turbine_id)
+    for con in continuous_list:
+        dirname = con.split('/')[-1]
+        output = os.path.join(output_path, dirname)
+        dfs = read_cfs(cfs, con, output, True)
+        dfs_cluster = pd.concat([df.set_index("C_TIME") for df in dfs.values()], join='inner', axis=1)
+        dfs_cluster.reset_index().to_csv(os.path.join(output, 'cluster_data.csv'), index=False)
+
+
+
+def paint_cluster_power(cluster_path, dfs):
+    dfs_cluster = pd.merge(dfs.values, axis=1, join='inner', on='C_TIME')
+    dfs = [pd.read_csv(os.path.join(cluster_path, file_path)).rename(columns={'C_ACTIVE_POWER':'power_'+file_path.split('/')[-1][-5], 'C_WS': 'C_WS_'+file_path.split('/')[-1][-5]}) for file_path in os.listdir(cluster_path)]
     df_cluster = pd.DataFrame({df.columns[-1]: df.iloc[:, -1] for df in dfs})
     df_cluster.insert(loc=0, column='C_TIME', value=dfs[0]['C_TIME'])
+    df_cluster.insert(loc=len(dfs)+1, column='SUM', value=df_cluster.iloc[:, 1:].sum(axis=1))
     df_cluster.to_csv(os.path.join(cluster_path, 'cluster_data.csv'), index=False)
 
 
@@ -96,4 +100,3 @@ def paint_cluster_power(cluster_path):
 
 
 
-

data_analysis.py

@@ -10,7 +10,7 @@
 import os.path
 
 import pandas as pd
-#from mpl_toolkits.basemap import Basemap
+# from mpl_toolkits.basemap import Basemap
 from scipy.signal import savgol_filter
 import numpy as np
 import matplotlib.pyplot as plt
@@ -106,7 +106,8 @@ class DataAnalysis:
         self.turbine = None
         # 风机的标号顺序
         self.turbine_id = list(range(102, 162))
-        self.turbine_id.remove(144)
+        b1b4 = [142, 143, 144, 145]
+        self.turbine_id = [id for id in self.turbine_id if id not in b1b4]
         # 风机功率数据15分钟级别
         self.power_15min = None
         # 风机经纬度信息
@@ -122,7 +123,7 @@ class DataAnalysis:
         # 计算风机功率差分
         self.compute_turbine_diff()
 
-    def load_data(self):
+    def load_data(self, normalize=False):
         """
         加载数据
         :return:
@@ -135,9 +136,21 @@ class DataAnalysis:
         #         power_15min.loc[i, 'C_REAL_VALUE'] = -34.56789
         # self.power_15min = power_15min
         turbine_path = '../data-process/data/output_filtered_csv_files/turbine-{}.csv'
-        self.turbine = {}
+        self.turbine, turbines = {}, []
         for i in self.turbine_id:
             self.turbine[i] = pd.read_csv(turbine_path.format(i))[20:].reset_index(drop=True)
+        if normalize:
+            self.normalize()
+
+    def normalize(self):
+        turbines = [self.turbine[i].values[:, 1:].astype(np.float32) for i in self.turbine_id]
+        turbines = np.vstack(turbines)
+        mean, std = np.mean(turbines, axis=0), np.std(turbines, axis=0)
+        for i in self.turbine_id:
+            c_time = self.turbine[i]['C_TIME']
+            self.turbine[i] = (self.turbine[i].iloc[:, 1:] - mean) / std
+            self.turbine[i].insert(loc=0, column='C_TIME', value=c_time)
+        return self.turbine
 
     def compute_turbine_diff(self):
         """
@@ -148,7 +161,7 @@ class DataAnalysis:
         ori_turbine_pic = []
         for turbine_i in self.turbine_id:
             diff_array = np.diff(
-                np.array(self.turbine[turbine_i]['C_ACTIVE_POWER'].values[self.data_start:self.data_end+1]))
+                np.array(self.turbine[turbine_i]['C_ACTIVE_POWER'].values[self.data_start:self.data_end + 1]))
             turbine_diff.append(diff_array)
             ori_turbine_pic.append(self.turbine[turbine_i]['C_ACTIVE_POWER'].values[self.data_start:self.data_end])
         self.ori_turbine_pic = ori_turbine_pic
@@ -311,7 +324,7 @@ class DataAnalysis:
         cmap = plt.get_cmap('viridis')
         for i, item in enumerate(self.cluster):
             if item == k:
-                pic_label.append('turbine-'+str(self.turbine_id[i]))
+                pic_label.append('turbine-' + str(self.turbine_id[i]))
                 y.append(self.ori_turbine_fft[i])
         for i in range(len(y)):
             color = cmap(i / 10)
@@ -383,12 +396,12 @@ class DataAnalysis:
         data_j = y[j - 1]
 
         plt.figure(figsize=(20, 10))
-        plt.plot(x, data_i, label='turbine {}'.format(self.turbine_id[i-1]), linestyle='solid')
-        plt.plot(x, data_j, label='turbine {}'.format(self.turbine_id[j-1]), linestyle='dashed')
+        plt.plot(x, data_i, label='turbine {}'.format(self.turbine_id[i - 1]), linestyle='solid')
+        plt.plot(x, data_j, label='turbine {}'.format(self.turbine_id[j - 1]), linestyle='dashed')
 
         plt.title('{} and {}'.format(i, j))
         plt.legend()
-        plt.savefig('analysis_img/{}_{}_turbine.png'.format(self.turbine_id[i-1], self.turbine_id[j-1]))
+        plt.savefig('analysis_img/{}_{}_turbine.png'.format(self.turbine_id[i - 1], self.turbine_id[j - 1]))
         plt.show()
 
     def process_ori_data(self):
@@ -410,14 +423,18 @@ class DataAnalysis:
             cluster = hierarchical_clustering(self.turbine_diff, threshold=1.4,
                                               similarity_func=compute_pearsonr)  # 层次聚类
         else:
-            cluster = hierarchical_clustering(data, threshold=1,
+            cluster = hierarchical_clustering(data, threshold=1.2,
                                               similarity_func=compute_pearsonr)
         self.cluster = cluster
+        # 在这里保存cluster变量
         from cluster_power import cluster_power_list_file, cluster_power_list_folder
 
-        output_path = '../data-process/data/cluester_power/'
-        cluster_power_list_file(self.cluster, self.turbine_id, input_path='../data-process/data/output_filtered_csv_files/', output_path=output_path)
-        cluster_power_list_folder(self.cluster, self.turbine_id, input_path='../data-process/data/continuous_data/', output_path=output_path)
+        output_path = '../data-process/data/cluster_power/'
+
+        cluster_power_list_file(self.cluster, self.turbine_id,
+                                input_path='../data-process/data/output_filtered_csv_files/', output_path=output_path)
+        cluster_power_list_folder(self.cluster, self.turbine_id, input_path='../data-process/data/continuous_data/',
+                                  output_path=output_path)
 
 
 data_analysis = DataAnalysis(data_length=9773,