package com.jiayue.biz.service.impl; import cn.hutool.core.io.FileUtil; import cn.hutool.core.text.csv.CsvData; import cn.hutool.core.text.csv.CsvReader; import cn.hutool.core.text.csv.CsvRow; import cn.hutool.core.text.csv.CsvUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.ZipUtil; import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.jiayue.biz.domain.*; import com.jiayue.biz.service.EmailService; import com.jiayue.biz.service.EmailWindTowerInfoService; import com.jiayue.biz.util.CalculationUtil; import com.jiayue.biz.util.CoordinateUtil; import com.jiayue.common.utils.file.FileUtils; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.poi.xssf.streaming.SXSSFRow; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.*; import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; /** * 采集数据解析实现类 * whc */ @Slf4j @Service public class AnalysisDataImpl { private static byte[] ZIP_HEADER_1 = new byte[]{80, 75, 3, 4}; private static byte[] ZIP_HEADER_2 = new byte[]{80, 75, 5, 6}; @Value("${file.outPutDir}") private String outPutDir; @Value("${file.excelDir}") private String excelDir; @Value("${file.outPutBackDir}") private String outPutBackDir; @Value("${file.excelBackDir}") private String excelBackDir; private String eolFilePath = "D:\\program\\systemFile\\"; private String eolDatafolder = "D:\\EOLdata"; private final WindTowerInfoServiceImpl windTowerInfoService; private final WindTowerDataParentTableServiceImpl windTowerDataParentTableService; private final EmailWindTowerInfoServiceImpl emailWindTowerInfoService; public AnalysisDataImpl(WindTowerInfoServiceImpl windTowerInfoService, WindTowerDataParentTableServiceImpl windTowerDataParentTableService, EmailWindTowerInfoServiceImpl emailWindTowerInfoService) { this.emailWindTowerInfoService = emailWindTowerInfoService; this.windTowerInfoService = windTowerInfoService; this.windTowerDataParentTableService = windTowerDataParentTableService; } /** * 先把outPutDir目录下的文件转成excel放到excelDir里 然后解析excel文件 读取数据存入数据库中 * * @throws InterruptedException */ public void analysisData(String type) throws InterruptedException { String path = outPutDir; File file = new File(path); if (!file.exists()) { file.mkdirs(); } File[] files = file.listFiles(); for (File f : files) { if (f.getName().contains(".log")) { f.delete(); } String[] xlsName = f.getName().split(".txt"); String excelFile = excelDir + xlsName[0] + ".xls"; HashMap hashMap = new HashMap<>(); //txt转xls if (type.equals("rld")) { analysisTXT(f.getPath(), excelFile); } else { hashMap = analysisRWDTXT(f.getPath(), excelFile); } log.info(f.getPath() + "文件已转换成xls移动到备份目录下"); //txt移动备份目录 File txtBackups = new File(outPutBackDir); if (!txtBackups.exists()) { txtBackups.mkdirs(); } FileUtil.move(f, txtBackups, true); //xls文件解析入库 if (type.equals("rld")) { analysisExcel(new File(excelFile), type); } else { analysisExcelRWD(new File(excelFile), hashMap, type); } log.info(xlsName[0] + ".xls" + "已转解析存入数据库并移动到备份目录下"); //xls移动备份目录 File xlsBackups = new File(excelBackDir); if (!xlsBackups.exists()) { xlsBackups.mkdirs(); } FileUtil.move(new File(excelFile), xlsBackups, true); } } /** * 解析文件 转成txt */ public HashMap analysisRWDTXT(String file_in, String file_out) { /*实例化输入、输出流*/ //输入文件 File f_in = new File(file_in); FileInputStream ips = null; InputStreamReader ipsr = null; //需要返回的map HashMap CHMap = new HashMap<>(); //输出文件 File f_out = new File(file_out); FileOutputStream ops = null; try { ips = new FileInputStream(f_in); ipsr = new InputStreamReader(ips); ops = new FileOutputStream(f_out); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } /*创建excel表格文件并输出*/ //创建工作薄 SXSSFWorkbook wb = new SXSSFWorkbook(); try { //创建新的一页 SXSSFSheet sheet = wb.createSheet("new sheet"); BufferedReader br = new BufferedReader(ipsr); String valueString = null; //表格的有效行计数 int count_line = 0; Boolean flag = false; String eqId = ""; String latitude = ""; String longitude = ""; String serialNumber = ""; String str = ""; String CH = ""; HashMap HeightMap = new HashMap<>(); String heightCheck = ""; StringBuilder dataName = new StringBuilder(); //存放ch和层高 ArrayList list = new ArrayList<>(); //循环处理每一行 while ((valueString = br.readLine()) != null) { if (valueString.contains("Channel #")) { String[] strs_cell = valueString.split("\t"); CH = "CH" + strs_cell[1]; } if (valueString.contains("Description\tNRG")) { String[] strs_cell = valueString.split("\t"); CHMap.put(CH, strs_cell[strs_cell.length - 1]); str = valueString; } if (valueString.contains("Height")) { String height = getNumberFromString(valueString); for (Map.Entry p : CHMap.entrySet()) { if (p.getKey().equals(CH)) { String h1 = p.getValue() + " " + height; CHMap.put(CH, h1); } } if (!height.isEmpty()) { //判断风速是否存在同样层高 if (str.contains("An") && heightCheck.contains(",")) { String[] split = heightCheck.split(","); for (String s : split) { if (height.equals(s)) { height = height + "A"; } } } if (str.contains("An")) { str = "An" + height; //把层高放入临时string heightCheck = heightCheck + height + ","; } HeightMap.put(str, height); } } if (valueString.contains("Site #")) { //测风塔编号 String[] strs_cell = valueString.split("\t"); eqId = strs_cell[1]; } if (valueString.contains("Latitude")) { //纬度 String[] strs_cell = valueString.split("\t"); String[] s = strs_cell[1].split(" "); String split = s[2].substring(0, 2); latitude = s[1] + " " + split; latitude = CoordinateUtil.changeToDu(latitude); } if (valueString.contains("Longitude")) { //经度 String[] strs_cell = valueString.split("\t"); String[] s = strs_cell[1].split(" "); String split = s[2].substring(0, 2); longitude = s[1] + " " + split; longitude = CoordinateUtil.changeToDu(longitude); } if (valueString.contains("Serial #")) { //记录仪编号 if (serialNumber.equals("")) { String[] strs_cell = valueString.split("\t"); serialNumber = strs_cell[1]; } } if (valueString.contains("Date & Time Stamp")) { flag = true; } if (flag) {//忽略开头代码 //分解每一行有效行 String[] strs_cell = valueString.split("\t");//用“|”切割每一行作为一个单元格数据 SXSSFRow row = sheet.createRow((short) count_line); for (int i = 0; i < strs_cell.length; ++i) {//循环每个切割得到的数据,除了最后一个(,,,)和第一个(空) //创建要显示的内容,创建一个单元格,第一个参数为列坐标,第二个参数为行坐标,第三个参数为内容 row.createCell((short) i).setCellValue(strs_cell[i]); //所有表头 if (!dataName.toString().equals("")) { dataName.append(","); dataName.append(strs_cell[i]); } else { dataName.append(strs_cell[i]); } } count_line++; } } toMap(CHMap); Set heightSet = new HashSet<>(); Set wdHeightSet = new HashSet<>(); StringBuffer height = new StringBuffer(); StringBuffer wdHeight = new StringBuffer(); //解析层高 for (Map.Entry name : HeightMap.entrySet()) { if (name.getKey().contains("An")) { String wsH = name.getValue(); heightSet.add(wsH + ","); } if (name.getKey().contains("V")) { String wdH = name.getValue(); wdHeightSet.add(wdH + ","); } } for (String s : heightSet) { height.append(s); } for (String s : wdHeightSet) { wdHeight.append(s); } //删除最后的逗号和不正常数据 height.deleteCharAt(height.toString().length() - 1); wdHeight.deleteCharAt(wdHeight.toString().length() - 1); String newHeight = height.toString().replace("-", ""); String newWdHeight = wdHeight.toString().replace("-", ""); if (newHeight.lastIndexOf(",") == newHeight.length() - 1) { newHeight = newHeight.substring(0, newHeight.length() - 1); } if (newWdHeight.lastIndexOf(",") == newWdHeight.length() - 1) { newWdHeight = newWdHeight.substring(0, newWdHeight.length() - 1); } //通过eqId 过滤出这个塔 List windTowerInfoList = windTowerInfoService.getByEquipmentNo(eqId); //为了防止两个塔编号相同的情况下 用编号+记录仪号来确定塔的唯一性 //如果这个塔的记录仪编号不等于空 则代表已经更新过记录了 List collect = windTowerInfoList.stream().sorted(Comparator.comparing(WindTowerInfo::getRecorderNo).reversed()).collect(Collectors.toList()); for (WindTowerInfo w : collect) { //为了防止两个塔编号相同的情况下 用编号+记录仪号来确定塔的唯一性 //如果这个塔的记录仪编号不等于空 则代表已经更新过记录了 if ("".equals(w.getRecorderNo()) || null == w.getRecorderNo()) { //如果这个塔的记录仪编号等于空 则直接更新塔的信息 w.setHeights(newHeight); w.setWdHeights(newWdHeight); w.setLatitude(new BigDecimal(latitude)); w.setLongitude(new BigDecimal(longitude)); w.setRecorderNo(serialNumber); windTowerInfoService.saveOrUpdate(w); } } //把创建的表格写入到输出流中,并关闭输出流 wb.write(ops); } catch ( IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (wb != null) { try { wb.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (ips != null) { try { ips.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (ops != null) { try { ops.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } return CHMap; } //以上是txt文件 //以下是excel入库文件 /** * 解析txt文件 输出成xls文件 * * @param file_in 输入文件 * @param file_out 输出文件 */ public void analysisTXT(String file_in, String file_out) { /*实例化输入、输出流*/ //输入文件 File f_in = new File(file_in); FileInputStream ips = null; InputStreamReader ipsr = null; //输出文件 File f_out = new File(file_out); FileOutputStream ops = null; try { ips = new FileInputStream(f_in); ipsr = new InputStreamReader(ips); ops = new FileOutputStream(f_out); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } /*创建excel表格文件并输出*/ //创建工作薄 SXSSFWorkbook wb = new SXSSFWorkbook(); try { //创建新的一页 SXSSFSheet sheet = wb.createSheet("new sheet"); BufferedReader br = new BufferedReader(ipsr); String valueString = null; //表格的有效行计数 int count_line = 0; Boolean flag = false; String eqId = ""; String latitude = ""; String longitude = ""; String serialNumber = ""; StringBuilder dataName = new StringBuilder(); //循环处理每一行 while ((valueString = br.readLine()) != null) { if (valueString.contains("Site Number")) { //测风塔编号 String[] strs_cell = valueString.split("\t"); eqId = strs_cell[1]; } if (valueString.contains("Latitude")) { //纬度 String[] strs_cell = valueString.split("\t"); latitude = strs_cell[1]; } if (valueString.contains("Longitude")) { //经度 String[] strs_cell = valueString.split("\t"); longitude = strs_cell[1]; } if (valueString.contains("Serial Number")) { //记录仪编号 if (serialNumber.equals("")) { String[] strs_cell = valueString.split("\t"); serialNumber = strs_cell[1]; } } if (flag) {//忽略开头代码 //分解每一行有效行 String[] strs_cell = valueString.split("\t");//用“|”切割每一行作为一个单元格数据 SXSSFRow row = sheet.createRow((short) count_line); for (int i = 0; i < strs_cell.length; ++i) {//循环每个切割得到的数据,除了最后一个(,,,)和第一个(空) //创建要显示的内容,创建一个单元格,第一个参数为列坐标,第二个参数为行坐标,第三个参数为内容 row.createCell((short) i).setCellValue(strs_cell[i]); //所有表头 if (!dataName.toString().equals("")) { dataName.append(","); dataName.append(strs_cell[i]); } else { dataName.append(strs_cell[i]); } } count_line++; } if (valueString.equals("Data")) { flag = true; } } //根据数据表头分解出层高 String[] names = dataName.toString().split(","); //风速 层高_风向 StringBuilder height = new StringBuilder(); //风向 层高_风向 StringBuilder wdHeight = new StringBuilder(); for (String name : names) { if (name.contains("Anem") || name.contains("Vane")) { String[] keys = name.split("_"); String b = getWsOrWd(keys[1], keys[5]); //120 String c = keys[2].substring(0, keys[2].length() - 4); //120_NNW String d = c + "_" + keys[3]; if (b.equals("ws")) { if (!height.toString().equals("")) { boolean flags = false; for (String h : height.toString().split(",")) { if (h.equals(d)) { flags = true; break; } } if (!flags) { height.append(","); height.append(d); } } else { height.append(d); } } else if (b.equals("wd")) { if (!wdHeight.toString().equals("")) { boolean flags = false; for (String h : wdHeight.toString().split(",")) { if (h.equals(d)) { flags = true; break; } } if (!flags) { wdHeight.append(","); wdHeight.append(d); } } else { wdHeight.append(d); } } } } //风速层高 如果一个层高有两个风速仪则第二个为层高A StringBuilder heights = new StringBuilder(); StringBuilder wdHeights = new StringBuilder(); for (String h : height.toString().split(",")) { if (heights.toString().equals("")) { //h=120_NNW h.split("_")[0]=120 heights.append(h.split("_")[0]); } else { boolean flags = false; //判断已添加层高里面是否有 h.split("_")[0] 这个层高 如果有返回true for (String hs : heights.toString().split(",")) { if (hs.equals(h.split("_")[0])) { flags = true; break; } } //如果有这个层高 则代表一个层高有两个风速仪 应为 120 120A if (flags) { heights.append(",").append(h.split("_")[0]).append("A"); } else { heights.append(",").append(h.split("_")[0]); } } } for (String h : wdHeight.toString().split(",")) { if (wdHeights.toString().equals("")) { //h=120_NNW h.split("_")[0]=120 wdHeights.append(h.split("_")[0]); } else { boolean flags = false; //判断已添加层高里面是否有 h.split("_")[0] 这个层高 如果有返回true for (String hs : wdHeights.toString().split(",")) { if (hs.equals(h.split("_")[0])) { flags = true; break; } } //如果有这个层高 则代表一个层高有两个风速仪 应为 120 120A if (flags) { wdHeights.append(",").append(h.split("_")[0]).append("A"); } else { wdHeights.append(",").append(h.split("_")[0]); } } } //通过eqId 过滤出这个塔 List windTowerInfoList = windTowerInfoService.getByEquipmentNo(eqId); List collect = windTowerInfoList.stream().sorted(Comparator.comparing(WindTowerInfo::getRecorderNo).reversed()).collect(Collectors.toList()); for (WindTowerInfo w : collect) { //为了防止两个塔编号相同的情况下 用编号+记录仪号来确定塔的唯一性 //如果这个塔的记录仪编号不等于空 则代表已经更新过记录了 if ("".equals(w.getRecorderNo()) || null == w.getRecorderNo()) { //如果这个塔的记录仪编号等于空 则直接更新塔的信息 w.setHeights(heights.toString()); w.setWdHeights(wdHeights.toString()); w.setLatitude(new BigDecimal(latitude)); w.setLongitude(new BigDecimal(longitude)); w.setRecorderNo(serialNumber); windTowerInfoService.saveOrUpdate(w); log.info("编号" + eqId + "测风塔信息更新成功"); } } //把创建的表格写入到输出流中,并关闭输出流 wb.write(ops); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (wb != null) { try { wb.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (ips != null) { try { ips.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (ops != null) { try { ops.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } } /** * @param file */ public void analysisExcelRWD(File file, HashMap CHMap, String type) { File file1 = new File("D:\\analysisData\\new\\"); if (!file1.exists()) { file1.mkdirs(); } ExcelReader reader = ExcelUtil.getReader(file, 0); List> read = reader.read(0); List objects = read.get(0); Map map = new HashMap<>(); CheckDataRecode checkDataRecode = new CheckDataRecode(); checkDataRecode.map.clear(); String fileNames = file.getName().substring(4, 12); for (Object object : objects) { //时间 if (object.equals("Date & Time Stamp")) { reader.addHeaderAlias(object.toString(), "time"); continue; } String numberFromString = getNumberFromString(object.toString()); int i = Integer.parseInt(numberFromString); String chValue = CHMap.get("CH" + numberFromString); String Splicing = object.toString() + " " + chValue; String[] keys = Splicing.toString().split(" "); //正常数据长度为6 if (keys.length <= 3) { continue; } //最大 最小 平均 标差 String a = getKeyStrRWD(keys[0]); //风速 风向 温 湿 压 String b = getWsOrWdRWD(keys[3], keys[4]); //层高 String c = keys[keys.length - 1]; String value = b + a; if (null != map.get(value + "_" + c)) { value = value + "_" + c + "A"; } else { if (!(b.equals("t") || b.equals("rh") || b.equals("pa"))) { value = value + "_" + Integer.parseInt(c); } } SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); try { checkDataRecode.map.put(value + "DeadDataStartTime", new DateTime(sdf.parse(fileNames).getTime())); } catch (ParseException e) { log.error("文件名日期转换异常"); } //判断该数据是否解析过 map.put(value, "1"); reader.addHeaderAlias(object.toString(), value); } List> allData = reader.readAll(); //数据筛选 List> filterData = new CheckDataRecode().checkValue(allData, type); String equipmentId = file.getName().substring(0, 4); String eTime = file.getName().substring(4, 12); windTowerDataParentTableService.packageData(filterData, equipmentId, type, ""); reader.close(); } public void analysisExcel(File file, String type) { File file1 = new File("D:\\analysisData\\new\\"); if (!file1.exists()) { file1.mkdirs(); } ExcelReader reader = ExcelUtil.getReader(file, 0); List> read = reader.read(0); List objects = read.get(0); Map map = new HashMap<>(); String[] fileNames = file.getName().split("_"); CheckDataRecode.map.clear(); //把表格里的数据转换成属性+层高方便解析入库 for (Object object : objects) { //时间 if (object.equals("Timestamp")) { reader.addHeaderAlias(object.toString(), "time"); continue; } String[] keys = object.toString().split("_"); //TODO改成传进去Ch1_Anem_140.00m_WNW_Avg_m/s 出来的是140风速的属性 //a = 最大 最小 平均值 标差 String a = getKeyStr(keys[4]); //b = 风速 风向 温 湿 压 String b = getWsOrWd(keys[1], keys[5]); //c = 层高 String c = keys[2].substring(0, keys[2].length() - 4); String value = b + a; if (null != map.get(value + "_" + c)) { value = value + "_" + Integer.parseInt(c) + "A"; } else { if (!(b.equals("t") || b.equals("rh") || b.equals("pa"))) { value = value + "_" + Integer.parseInt(c); } } //判断该数据是否解析过 map.put(value, "1"); CheckDataRecode.map.put(value + "DeadDataStartTime", new DateTime(fileNames[1])); reader.addHeaderAlias(object.toString(), value); } List> allData = reader.readAll(); //数据筛选 List> filterData = new CheckDataRecode().checkValue(allData, type); String eqId = file.getName().split("_")[0]; String eTime = file.getName().split("_")[1]; windTowerDataParentTableService.packageData(filterData, eqId, type, ""); reader.close(); } //拼装实体 private String getKeyStr(String key) { if (("Avg").equals(key)) { return "Ave"; } if (("SD").equals(key)) { return "Sta"; } return key; } //拼装实体 private String getWsOrWd(String key, String key2) { if (("Anem").equals(key)) { return "ws"; } if (("Vane").equals(key)) { return "wd"; } if (("Analog").equals(key)) { if (key2.equals("C")) { return "t"; } if (key2.equals("mb") || key2.equals("hPa") || key2.equals("kpa") || key2.equals("KPa")) { return "pa"; } if (key2.equals("%RH")) { return "rh"; } } return key; } public Map toMap(Map map) { return map; } private String getKeyStrRWD(String key) { if (key.contains("Avg")) { return "Ave"; } if (key.contains("SD")) { return "Sta"; } if (key.contains("Max")) { return "Max"; } if (key.contains("Min")) { return "Min"; } return key; } private String getWsOrWdRWD(String key, String key2) { if (key.contains("An") || key2.equals("em")) { return "ws"; } if (key.contains("V")) { return "wd"; } if (key2.contains("C") || key2.contains("T")) { return "t"; } if (key2.contains("mb") || key2.contains("hPa") || key2.contains("kpa") || key2.contains("kPa")) { return "pa"; } if (key2.contains("%RH")) { return "rh"; } return key; } /** * @param file csv文件 * @param eqId 设备Id */ public void acousticRadar(File file, String eqId) { List> mapList = new ArrayList<>(); HashMap> wsStaMapList = new HashMap<>(); CsvReader reader = CsvUtil.getReader(); //解析csv文件设置编码为GBK CsvData data = reader.read(file, Charset.forName("gbk")); //获取到csv文件 List rows = data.getRows(); //获取层高 List equipmentNo = windTowerInfoService.lambdaQuery().eq(WindTowerInfo::getEquipmentNo, eqId).list(); String height = ""; if (equipmentNo.size() == 0) { CsvRow csvRow = rows.get(0); StringBuilder heights = new StringBuilder(); for (int i = csvRow.size() - 1; i >= 0; i--) { if (csvRow.get(i).contains("高度")) { if (heights.length() != 0) { heights.append(","); } heights.append(CalculationUtil.getNumberFromString(csvRow.get(i))); } } height = heights.toString(); WindTowerInfo windTowerInfo = new WindTowerInfo(); windTowerInfo.setEquipmentNo(eqId); windTowerInfo.setRecorderNo(eqId); windTowerInfo.setHeights(height); windTowerInfo.setWdHeights(height); windTowerInfoService.saveWindTowerInfo(windTowerInfo); } else { WindTowerInfo windTowerInfo = equipmentNo.get(0); height = windTowerInfo.getHeights(); } int total = 0; String str = ""; boolean firstBoolean = true; //遍历表头 设置表头 try { for (String s : rows.get(0)) { if (s.contains("高度")) { str = CalculationUtil.getNumberFromString(s); } if (s.contains("风速")) { rows.get(0).set(total, "ws_Ave" + str); } if (s.contains("风向")) { rows.get(0).set(total, "wd_Ave" + str); } if (s.contains("时间")) { rows.get(0).set(total, "time"); } if (s.contains("温度")) { rows.get(0).set(total, "t_ave"); } if (s.contains("湿度")) { rows.get(0).set(total, "rh_ave"); } if (s.contains("压力")) { rows.get(0).set(total, "pa_ave"); } total++; } //临时变量 boolean f = false; SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm"); CheckDataRecode.map.clear(); String time1 = ""; //父表入库 for (CsvRow row : rows) { //跳过表头 if (!f) { f = true; continue; } HashMap hashMap = new HashMap<>(); // HashMap firstMap = new HashMap<>(); if (rows.get(0).contains("time")) { int time = rows.get(0).indexOf("time"); if (!row.get(time).isEmpty()) { time1 = row.get(time); //异常数据 if (time1.contains("000000000000")) { continue; } } } for (String h : height.split(",")) { BigDecimal wsAve = conversion("ws_Ave" + h, row, rows); if (wsStaMapList.get(h) == null) { ArrayList wsStaList = new ArrayList<>(); wsStaList.add(wsAve.doubleValue()); wsStaMapList.put(h, wsStaList); } else { wsStaMapList.get(h).add(wsAve.doubleValue()); } } if ((sdf.parse(time1).getMinutes() % 10) == 0) { hashMap.put("time", time1); BigDecimal tAve = conversion("t_ave", row, rows); hashMap.put("tAve", tAve); BigDecimal rhAve = conversion("rh_ave", row, rows); hashMap.put("rhAve", rhAve); BigDecimal paAve = conversion("pa_ave", row, rows); hashMap.put("paAve", paAve); BigDecimal airDensity = BigDecimal.valueOf(1.293).multiply(new BigDecimal(273).divide(new BigDecimal(273).add(CalculationUtil.getBigDecimal(tAve)), 2, RoundingMode.HALF_UP)).multiply(CalculationUtil.getBigDecimal(paAve)).divide(new BigDecimal(1013), 2, RoundingMode.HALF_UP); hashMap.put("airDensity", airDensity); for (Map.Entry> entry : wsStaMapList.entrySet()) { BigDecimal wsAve = conversion("ws_Ave" + entry.getKey(), row, rows); hashMap.put("wsAve_" + entry.getKey(), wsAve); BigDecimal wdAve = conversion("wd_Ave" + entry.getKey(), row, rows); hashMap.put("wdAve_" + entry.getKey(), wdAve); hashMap.put("wsSta_" + entry.getKey(), CalculationUtil.calculateSD(entry.getValue())); } // if(firstBoolean){ // firstMap = hashMap; // firstBoolean = false; // } mapList.add(hashMap); wsStaMapList = new HashMap<>(); } } //过滤数据 List> mapList1 = new CheckDataRecode().checkValue(mapList, "sld"); windTowerDataParentTableService.packageData(mapList1, eqId, "sld", ""); com.jiayue.biz.util.FileUtil.move(file.getPath(), eolFilePath + File.separator + eqId); } catch (Exception e) { e.printStackTrace(); } } //根据表头位置找到对应数据 换算 public BigDecimal conversion(String filedName, CsvRow row, List rows) { BigDecimal data = BigDecimal.ZERO; if (rows.get(0).contains(filedName)) { int dataTotal = rows.get(0).indexOf(filedName); if (!row.get(dataTotal).isEmpty()) { String windData = row.get(dataTotal); if (filedName.contains("ws")) { data = CalculationUtil.getBigDecimal(windData).divide(BigDecimal.valueOf(100), 2, RoundingMode.HALF_UP); } else { data = CalculationUtil.getBigDecimal(windData); } } } return data; } //判断是否是压缩包 public static boolean isArchiveFile(File file) { if (file == null) { return false; } if (file.isDirectory()) { return false; } boolean isArchive = false; InputStream input = null; try { input = new FileInputStream(file); byte[] buffer = new byte[4]; int length = input.read(buffer, 0, 4); if (length == 4) { isArchive = (Arrays.equals(ZIP_HEADER_1, buffer)) || (Arrays.equals(ZIP_HEADER_2, buffer)); } } catch (IOException e) { e.printStackTrace(); } finally { if (input != null) { try { input.close(); } catch (IOException e) { } } } return isArchive; } /** * 从 String 中提取数字 * * @param string * @return */ public static String getNumberFromString(String string) { String str = string; str = str.trim(); StringBuffer str2 = new StringBuffer(); if (str != null && !"".equals(str)) { for (int i = 0; i < str.length(); i++) { if (str.charAt(i) >= 48 && str.charAt(i) <= 57) { String s = String.valueOf(str.charAt(i)); str2.append(s); } } } return str2.toString(); } public void moveEolFileAndParseEol() { File file = new File(eolDatafolder); File[] files = file.listFiles(); try { for (File file1 : files) { if (file1.isDirectory()) { File[] files1 = file1.listFiles(); //筛选包含.log后缀的文件 List fileNameList = Arrays.asList(files1).stream().filter(f -> !f.isDirectory() && f.getName().substring(f.getName().lastIndexOf(".")).contains(".log")).collect(Collectors.toList()); //循环文件名 for (File fileNameForELog : fileNameList) { String fileName = fileNameForELog.getName().substring(0, fileNameForELog.getName().indexOf(".log")); //找出wnd文件 List wndFile = Arrays.stream(files1).filter(f -> f.getName().contains(".wnd") && f.getName().contains(fileName)).collect(Collectors.toList()); if (wndFile.size() > 0) { HashMap stringStringHashMap = this.parseWnd(wndFile.get(0)); String eqNo = stringStringHashMap.get("SiteNumber"); //找出csv文件 List csvFile = Arrays.stream(files1).filter(f -> f.getName().contains(".csv") && f.getName().contains(fileName)).collect(Collectors.toList()); if (csvFile.size() > 0 && wndFile.size() > 0) { parseEol(csvFile.get(0), stringStringHashMap, eqNo, fileNameForELog); } } } } } } catch (Exception e) { e.printStackTrace(); } } //解析eol文件 public void parseEol(File fileCsv, HashMap stringStringHashMap, String eqNo, File fileNameForELog) { //todo 需要修改 // String s = "C:\\Users\\Administrator\\Desktop\\ID220810_20220919_180304_20230515_063537.log_20230515_063540.csv"; // //从文件中读取CSV数据 // File file = FileUtil.file(s); CsvReader reader = CsvUtil.getReader(); CsvData data = reader.read(fileCsv); List rows = data.getRows(); //todo 不严谨但是能用 CsvRow row = data.getRow(2); HashSet wsHeight = new HashSet<>(); HashSet wdHeight = new HashSet<>(); boolean state = true; //遍历行 List> dataList = new ArrayList<>(); for (CsvRow csvRow : rows) { //getRawList返回一个List列表,列表的每一项为CSV中的一个单元格(既逗号分隔部分) HashMap hashMap = new HashMap<>(); for (int i = 0; i < csvRow.size(); i++) { String str = csvRow.get(i); if (str.contains("DateTime")) { if (state) { state = false; break; } } String s1 = this.assembleWind(str, wsHeight, wdHeight); //修改表头 csvRow.set(i, s1); hashMap.put(row.get(i), str); } //过滤数据 if (hashMap.size() > 3 && !hashMap.get("time").equals("DateTime")) { dataList.add(hashMap); } } this.emailWindTowerInfoSave(eqNo); List list = windTowerInfoService.list(); List collect = list.stream().filter(w -> w.getEquipmentNo().equals(eqNo)).collect(Collectors.toList()); if (collect.size() > 0) { if (collect.get(0).getRecorderNo() == null) { String wsH = this.heightReverseOrder(wsHeight); String wdH = this.heightReverseOrder(wdHeight); WindTowerInfo windTowerInfo = new WindTowerInfo(); windTowerInfo.setEquipmentNo(eqNo); windTowerInfo.setName(eqNo); windTowerInfo.setId(collect.get(0).getId()); windTowerInfo.setType("email"); windTowerInfo.setHeights(wsH); windTowerInfo.setWdHeights(wdH); windTowerInfo.setLatitude(CalculationUtil.getBigDecimal(stringStringHashMap.get("Latitude"))); windTowerInfo.setLongitude(CalculationUtil.getBigDecimal(stringStringHashMap.get("Longitude"))); windTowerInfo.setRecorderNo(stringStringHashMap.get("SiteNumber")); windTowerInfo.setModelNumber("eol"); windTowerInfoService.saveOrUpdate(windTowerInfo); } else { try { //移动原始文件 com.jiayue.biz.util.FileUtil.move(fileNameForELog.getPath(), eolFilePath + File.separator + eqNo); //移动csv文件 // com.jiayue.biz.util.FileUtil.move(fileCsv.getPath(), eolFilePath + File.separator + eqNo); } catch (Exception e) { e.printStackTrace(); } windTowerDataParentTableService.packageData(dataList, eqNo, "eol", ""); } } } //解析经纬度 记录仪编号 解析wnd文件 public HashMap parseWnd(File wndFile) { String fileParent = wndFile.getParent(); String fileName = wndFile.getName(); String xls = fileName.replace(".wnd", ".csv"); CsvReader reader = CsvUtil.getReader(); //从文件中读取CSV数据 File file = FileUtil.file(fileParent + File.separator + fileName); file.renameTo(new File(fileParent + File.separator + xls)); CsvData data = reader.read(new File(fileParent + File.separator + xls)); CsvRow row = data.getRow(1); String Longitude = ""; String Latitude = ""; String SiteNumber = ""; HashMap hashMap = new HashMap<>(); for (String s1 : row) { if (s1.contains("Longitude")) { Longitude = CalculationUtil.getNumberFromStringForDian(s1); hashMap.put("Longitude", Longitude); } else if (s1.contains("Latitude")) { Latitude = CalculationUtil.getNumberFromStringForDian(s1); hashMap.put("Latitude", Latitude); } else if (s1.contains("SiteNumber")) { SiteNumber = CalculationUtil.getNumberFromString(s1); hashMap.put("SiteNumber", SiteNumber); } } return hashMap; } public void emailWindTowerInfoSave(String equipmentNo) { List emailWindTowerInfoList = emailWindTowerInfoService.list(); List emailWindTowerInfos = emailWindTowerInfoList.stream().filter(e -> e.getEquipmentNo().equals(equipmentNo)).collect(Collectors.toList()); if (emailWindTowerInfos.size() == 0) { EmailWindTowerInfo emailWindTowerInfo = new EmailWindTowerInfo(); emailWindTowerInfo.setId(UUID.randomUUID().toString()); emailWindTowerInfo.setEquipmentNo(equipmentNo); } } public String heightReverseOrder(HashSet Height) { StringBuilder builder = new StringBuilder(); List wsCollect = Height.stream().map(Integer::parseInt).sorted(Comparator.reverseOrder()).map(String::valueOf).collect(Collectors.toList()); for (String h : wsCollect) { if (builder.length() > 0) { builder.append(","); } builder.append(h); } return builder.toString(); } public String assembleWind(String s, HashSet wsHeight, HashSet wdHeight) { String str = s; if (s.contains("WS")) { String wsAndType = s.substring(s.indexOf("WS")); //层高 String height = ""; if (!s.contains("TI")) { height = CalculationUtil.getNumberFromString(s.substring(s.indexOf("WS"))); wsHeight.add(height); } //ave/min/max/sta String type = this.assembleProperty(wsAndType); str = "ws" + type + "_" + height; } else if (s.contains("WD")) { String wdAndType = s.substring(s.indexOf("WD")); //层高 String height = CalculationUtil.getNumberFromString(wdAndType); wdHeight.add(height); //ave/min/max/sta String type = this.assembleProperty(wdAndType); str = "wd" + type + "_" + height; } else if (s.contains("DateTime")) { str = "time"; } else if (s.contains("PR")) { String type = this.assembleProperty(s); str = "pa" + type; } else if (s.contains("TEM")) { String type = this.assembleProperty(s); str = "t" + type; } return str; } public String assembleProperty(String property) { String s = ""; if (property.contains("-")) { if (property.contains("STDev")) { s = "Sta"; } else if (property.contains("Max")) { s = "Max"; } else if (property.contains("Min")) { s = "Min"; } } else { s = "Ave"; } return s; } public boolean windTowerInfoType(String id) { List equipmentNo = windTowerInfoService.getByEquipmentNo(id); List emailWindTowerInfos = emailWindTowerInfoService.lambdaQuery().eq(EmailWindTowerInfo::getEquipmentNo, id).list(); if (equipmentNo.size() != 0 && emailWindTowerInfos.size() != 0) { return true; } else { if (emailWindTowerInfos.size() == 0) { //如果测风塔里面没有这个塔并且邮件测风塔里也没有这个塔 则添加到邮箱读取的测风塔表中 EmailWindTowerInfo emailWindTowerInfo = new EmailWindTowerInfo(); emailWindTowerInfo.setEquipmentNo(id); emailWindTowerInfoService.save(emailWindTowerInfo); log.info(id + " 测风塔信息里没有这个的塔 并且邮件测风塔里也没有这个塔"); } return false; } } /** * dat文件解析 */ public void parseByDat(File file) { String dec = "D:\\in" + File.separator; String fileName = file.getName(); File newFile = new File(file.getPath()); //修改后缀名 方便解析 if (fileName.contains(".dat")) { String newFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".csv"; newFile = new File(dec + newFileName); file.renameTo(newFile); } CsvReader reader = CsvUtil.getReader(); CsvData csvRows = reader.read(newFile); List rows = csvRows.getRows(); CsvRow rowsOne = rows.get(1); CsvRow rowsF = rows.get(5); StringBuilder height = new StringBuilder(); StringBuilder wdHeight = new StringBuilder(); WindTowerInfo windTowerInfo = new WindTowerInfo(); for (int i = rowsOne.size() - 1; i >= 0; i--) { String wsOrWdOrTOrRhOrPa = getWsOrWdOrTOrRhOrPa(rowsOne.get(i), height, wdHeight, i, rowsF, windTowerInfo); rowsOne.set(i, wsOrWdOrTOrRhOrPa); } List equipmentNo = windTowerInfoService.getByEquipmentNo(windTowerInfo.getEquipmentNo()); if (windTowerInfoType(windTowerInfo.getEquipmentNo())) { if (equipmentNo.size() == 0) { windTowerInfo.setHeights(height.toString()); windTowerInfo.setWdHeights(height.toString()); windTowerInfoService.saveOrUpdate(windTowerInfo); } List> mapList = parseByHead(rows, rowsOne); List> mapList1 = new CheckDataRecode().checkValue(mapList, "dat"); windTowerDataParentTableService.packageData(mapList1, windTowerInfo.getEquipmentNo(), "dat", ""); //将后缀名修改回来 String newFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".dat"; File file1 = new File(dec + newFileName); newFile.renameTo(file1); try { com.jiayue.biz.util.FileUtil.move(file1.getPath(), eolFilePath + File.separator + windTowerInfo.getEquipmentNo()); } catch (Exception e) { log.info("dat文件转移失败"); throw new RuntimeException(e); } } } public List> parseByHead(List rows, CsvRow rowsOne) { List> mapList = new ArrayList<>(); for (int i = 4; i < rows.size(); i++) { HashMap hashMap = new HashMap<>(); for (int j = 0; j < rows.get(i).size(); j++) { if (StrUtil.isNotBlank(rowsOne.get(j))) { if (rowsOne.get(j).equals("time")) { hashMap.put(rowsOne.get(j), rows.get(i).get(j)); } else { hashMap.put(rowsOne.get(j), new BigDecimal(rows.get(i).get(j))); } } } mapList.add(hashMap); } return mapList; } public String getWsOrWdOrTOrRhOrPa(String attribute, StringBuilder wsHeight, StringBuilder wdHeight, int i, CsvRow rowsF, WindTowerInfo windTowerInfo) { String str = ""; String height = ""; String avgOrStaOrMaxOrMin = getAvgOrStaOrMaxOrMin(attribute); if (attribute.contains("TIMESTAMP")) { str = "time"; } else if (attribute.contains("WS")) { height = CalculationUtil.getNumberFromString(attribute); if (wsHeight.toString().equals("")) { wsHeight.append(height); } else { if (!wsHeight.toString().contains(height)) { wsHeight.append(",").append(height); } } return "ws" + avgOrStaOrMaxOrMin + "_" + height; } else if (attribute.contains("WD")) { height = CalculationUtil.getNumberFromString(attribute); if (wdHeight.toString().equals("")) { wdHeight.append(height); } else { if (!wdHeight.toString().contains(height)) { wdHeight.append(",").append(height); } } return "wd" + avgOrStaOrMaxOrMin + "_" + height; } else if (attribute.contains("Ta")) { str = "t"; } else if (attribute.contains("RH")) { str = "rh"; } else if (attribute.contains("pa")) { str = "pa"; } else if (attribute.contains("Station")) { windTowerInfo.setEquipmentNo(rowsF.get(i)); } else if (attribute.equals("N")) { windTowerInfo.setLatitude(new BigDecimal(rowsF.get(i))); } else if (attribute.equals("E")) { windTowerInfo.setLongitude(new BigDecimal(rowsF.get(i))); } else { return ""; } return str + avgOrStaOrMaxOrMin; } public String getAvgOrStaOrMaxOrMin(String attribute) { String str = ""; if (attribute.contains("Batt") || attribute.contains("Minute")) { return str; } if (attribute.contains("Avg")) { str = "Avg"; } else if (attribute.contains("Std")) { str = "Sta"; } else if (attribute.contains("Max")) { str = "Max"; } else if (attribute.contains("Min")) { str = "Min"; } return str; } }