|
@@ -0,0 +1,270 @@
|
|
|
+package com.jiayue.insu.incloud.pulldata;
|
|
|
+
|
|
|
+import cn.hutool.core.collection.CollectionUtil;
|
|
|
+import cn.hutool.core.date.DateUtil;
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import cn.hutool.http.HttpRequest;
|
|
|
+import cn.hutool.http.HttpUtil;
|
|
|
+import cn.hutool.json.JSONArray;
|
|
|
+import cn.hutool.json.JSONObject;
|
|
|
+import cn.hutool.json.JSONUtil;
|
|
|
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|
|
+import com.jiayue.insu.incloud.constants.CommonStant;
|
|
|
+import com.jiayue.insu.incloud.constants.enums.StatusEnum;
|
|
|
+import com.jiayue.insu.incloud.constants.vo.FileCreateLog;
|
|
|
+import com.jiayue.insu.incloud.entity.ForecastData;
|
|
|
+import com.jiayue.insu.incloud.entity.Record;
|
|
|
+import com.jiayue.insu.incloud.entity.Station;
|
|
|
+import com.jiayue.insu.incloud.service.ForecastDataService;
|
|
|
+import com.jiayue.insu.incloud.service.RecordService;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import java.io.*;
|
|
|
+import java.math.BigDecimal;
|
|
|
+import java.math.RoundingMode;
|
|
|
+import java.text.SimpleDateFormat;
|
|
|
+import java.time.LocalDateTime;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.Comparator;
|
|
|
+import java.util.Date;
|
|
|
+import java.util.List;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+@Component
|
|
|
+@Slf4j
|
|
|
+public class PullCorrectData implements IPullInitForecastData{
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private RecordService recordService;
|
|
|
+ @Autowired
|
|
|
+ private ForecastDataService forecastDataService;
|
|
|
+ @Value("${pull.url}")
|
|
|
+ String url;
|
|
|
+ @Value("${pull.fileurl}")
|
|
|
+ String fileurl;
|
|
|
+ @Value("${pull.fileDir}")
|
|
|
+ String fileDir;
|
|
|
+ @Override
|
|
|
+ public Boolean pullDQAndNWPData(Station station) {
|
|
|
+ boolean result = false;
|
|
|
+ Record record = new Record();
|
|
|
+ LocalDateTime localDateTime = DateUtil.date().toLocalDateTime();
|
|
|
+ record.setStationCode(station.getStationCode());
|
|
|
+ record.setInCode(station.getInCode());
|
|
|
+ record.setTime(localDateTime);
|
|
|
+ record.setType(CommonStant.RECORD_TYPE_PULL);
|
|
|
+ log.info("下载minio原始RB文件 --> {}"+station.getStationCode());
|
|
|
+ try {
|
|
|
+ HttpRequest httpRequest = HttpRequest.get(url + station.getStationCode());
|
|
|
+ httpRequest.setGlobalTimeout(20000);
|
|
|
+ String body = httpRequest.execute().body();
|
|
|
+ JSONObject json = JSONUtil.parseObj(body);
|
|
|
+ String code = json.get("code").toString();
|
|
|
+ String data = json.get("data").toString();
|
|
|
+ if (code.equals("0") && data.length() > 0) {
|
|
|
+ JSONArray array = JSONUtil.parseArray(data);
|
|
|
+ List<FileCreateLog> list = array.toList(FileCreateLog.class);
|
|
|
+ List<FileCreateLog> dqListNow = new ArrayList<>();
|
|
|
+ List<FileCreateLog> nwpListNow = new ArrayList<>();
|
|
|
+ String dateNow = DateUtil.format(new Date(), "yyyyMMdd");
|
|
|
+ if (CollectionUtil.isNotEmpty(list)) {
|
|
|
+ dqListNow = list.stream().filter(f -> f.getFileName().contains("DQ_" + dateNow)).collect(Collectors.toList());
|
|
|
+ nwpListNow = list.stream().filter(f -> f.getFileName().contains("NWP_" + dateNow)).collect(Collectors.toList());
|
|
|
+ if (CollectionUtil.isNotEmpty(dqListNow) && CollectionUtil.isNotEmpty(nwpListNow)) {
|
|
|
+ dqListNow.sort(Comparator.comparing(FileCreateLog::getFileName).reversed());
|
|
|
+ nwpListNow.sort(Comparator.comparing(FileCreateLog::getFileName).reversed());
|
|
|
+
|
|
|
+ File dqNowFile = null;
|
|
|
+ File nwpNowFile = null;
|
|
|
+ List<ForecastData> listNow = new ArrayList<>();
|
|
|
+ String fileSaveUrl = fileDir + File.separatorChar + station.getStationCode();
|
|
|
+ log.info(fileSaveUrl);
|
|
|
+ File dir = new File(fileSaveUrl);
|
|
|
+ if (!dir.exists()) {// 判断目录是否存在
|
|
|
+ dir.mkdirs();
|
|
|
+ log.info("创建目录:" + fileSaveUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ // String dqNowUrl = fileurl + dqListNow.get(0).getId();
|
|
|
+ String dqNowUrl = "https://117.78.19.70:9010/client/getFileById?id=" + dqListNow.get(0).getId();
|
|
|
+ dqNowFile = HttpUtil.downloadFileFromUrl(dqNowUrl, fileSaveUrl + File.separatorChar +dqListNow.get(0).getFileName());
|
|
|
+ //String nwpNowUrl = fileurl + nwpListNow.get(0).getId();
|
|
|
+ String nwpNowUrl = "https://117.78.19.70:9010/client/getFileById?id=" + nwpListNow.get(0).getId();
|
|
|
+ nwpNowFile = HttpUtil.downloadFileFromUrl(nwpNowUrl, fileSaveUrl + File.separatorChar + nwpListNow.get(0).getFileName());
|
|
|
+
|
|
|
+ if (dqNowFile != null && nwpNowFile != null) {
|
|
|
+ //解析
|
|
|
+ listNow = fileAnalysis(dqNowFile, nwpNowFile, station,localDateTime);
|
|
|
+
|
|
|
+ if (CollectionUtil.isNotEmpty(listNow)) {
|
|
|
+
|
|
|
+ //删除该时间段内数据
|
|
|
+ LambdaQueryWrapper<ForecastData> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
|
|
+ lambdaQueryWrapper.between(ForecastData::getForecastTime,listNow.get(0).getForecastTime(), listNow.get(listNow.size() - 1).getForecastTime());
|
|
|
+ forecastDataService.remove(lambdaQueryWrapper);
|
|
|
+ forecastDataService.saveBatch(listNow);
|
|
|
+ String fileName = dqNowFile.getName() + "," + nwpNowFile.getName();
|
|
|
+ record.setContent(fileName);
|
|
|
+ record.setState(StatusEnum.SUCCESS.getCode());
|
|
|
+ result = true;
|
|
|
+ log.info(station.getStationCode() + " 存入初始化数据:" + listNow.size() + "条");
|
|
|
+ } else {
|
|
|
+ record.setState(StatusEnum.DOWNLOAD_FILE_ERROR.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.DOWNLOAD_FILE_ERROR.getMsg());
|
|
|
+ }
|
|
|
+
|
|
|
+ } else {
|
|
|
+ record.setState(StatusEnum.DOWNLOAD_FILE_ERROR.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.DOWNLOAD_FILE_ERROR.getMsg());
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ } catch (Exception e) {
|
|
|
+ record.setState(StatusEnum.DOWNLOAD_FILE_FAIL.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.DOWNLOAD_FILE_FAIL.getMsg());
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+
|
|
|
+ } else {
|
|
|
+
|
|
|
+ record.setState(StatusEnum.FILE_NULL.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.FILE_NULL.getMsg());
|
|
|
+
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ record.setState(StatusEnum.FILE_NULL.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.FILE_NULL.getMsg());
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ } else {
|
|
|
+ record.setState(StatusEnum.RESPONSE_FAIL.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.RESPONSE_FAIL.getMsg());
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ record.setState(StatusEnum.RESPONSE_FAIL.getCode());
|
|
|
+ log.error("下载minio原始RB文件 --> {} 失败,失败原因:{}",station.getStationCode(),StatusEnum.RESPONSE_FAIL.getMsg());
|
|
|
+ e.printStackTrace();
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ recordService.save(record);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ public List<ForecastData> fileAnalysis(File dqFile, File nwpFile, Station station, LocalDateTime localDateTime) {
|
|
|
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
|
|
+ List<ForecastData> aioList = new ArrayList<>();
|
|
|
+ List<ForecastData> resultList = new ArrayList<>();
|
|
|
+ boolean jx = true;
|
|
|
+ if (dqFile.renameTo(dqFile)) {
|
|
|
+ InputStreamReader dqRead = null;
|
|
|
+ BufferedReader dqBufferedReader = null;
|
|
|
+ String stringLine;
|
|
|
+ ForecastData stf;
|
|
|
+ try {
|
|
|
+ dqRead = new InputStreamReader(new FileInputStream(dqFile), "utf-8");
|
|
|
+ dqBufferedReader = new BufferedReader(dqRead);
|
|
|
+ while ((stringLine = dqBufferedReader.readLine()) != null) {
|
|
|
+ String[] string_arr = stringLine.split("\t");
|
|
|
+ if (string_arr.length == 4 && string_arr[0].startsWith("#")) {
|
|
|
+ if (StrUtil.isNotEmpty(string_arr[2])) {
|
|
|
+ stf = new ForecastData();
|
|
|
+ stf.setFpValue(new BigDecimal(string_arr[3] + ""));
|
|
|
+ stf.setForecastTime(sdf.parse(string_arr[2]).getTime());
|
|
|
+ stf.setInitTime(localDateTime);
|
|
|
+ stf.setStationCode(station.getStationCode());
|
|
|
+ stf.setInCode(station.getSignCode());
|
|
|
+ aioList.add(stf);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ jx = false;
|
|
|
+ log.error(station.getStationCode()+" 解析当天DQ文件错误:", e);
|
|
|
+ } finally {
|
|
|
+ close(dqBufferedReader, dqRead);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (CollectionUtil.isNotEmpty(aioList)) {
|
|
|
+ if (nwpFile.renameTo(nwpFile)) {
|
|
|
+ InputStreamReader readNwp = null;
|
|
|
+ BufferedReader bufferedReaderNwp = null;
|
|
|
+ try {
|
|
|
+ readNwp = new InputStreamReader(new FileInputStream(nwpFile), "utf-8");//考虑到编码格式
|
|
|
+ bufferedReaderNwp = new BufferedReader(readNwp);
|
|
|
+ String fileName = nwpFile.getName();
|
|
|
+ fileName = fileName.substring(fileName.indexOf("_") + 1, fileName.lastIndexOf("."));
|
|
|
+ String lineTxt;
|
|
|
+ BigDecimal nwpDirectRadiation = new BigDecimal(0.7); //直接辐射
|
|
|
+ BigDecimal nwpDiffuseRadiation = new BigDecimal(0.3); //散接辐射
|
|
|
+
|
|
|
+ while ((lineTxt = bufferedReaderNwp.readLine()) != null) {
|
|
|
+ //NWP文件按照Tab方式截取
|
|
|
+ String[] datas = lineTxt.split("\t");
|
|
|
+ if (datas.length == 35 && datas[0].startsWith("#")) {
|
|
|
+ if (datas != null && datas.length > 0) {
|
|
|
+ Long finalTime = sdf.parse(datas[5]).getTime();
|
|
|
+ List<ForecastData> filterAios = aioList.stream().filter(a -> a.getForecastTime().longValue() == finalTime.longValue()).collect(Collectors.toList());
|
|
|
+ if (CollectionUtil.isNotEmpty(filterAios)) {
|
|
|
+ ForecastData aio = filterAios.get(0);
|
|
|
+ aio.setTemperature(new BigDecimal(datas[6]).subtract(new BigDecimal(273.15)).setScale(2, RoundingMode.HALF_UP));//温度
|
|
|
+ aio.setSwr(new BigDecimal(datas[9]).setScale(2, RoundingMode.HALF_UP));//短波辐射(相当于总辐射)
|
|
|
+ aio.setPressure(new BigDecimal(datas[8]).setScale(2, RoundingMode.HALF_UP));//地表气压
|
|
|
+ aio.setHumidity(new BigDecimal(datas[7]).setScale(2, RoundingMode.HALF_UP));//2m相对湿度
|
|
|
+ aio.setDiffuseRadiation(new BigDecimal(datas[9]).multiply(nwpDiffuseRadiation).setScale(2, RoundingMode.HALF_UP));//散接辐射
|
|
|
+ aio.setDirectRadiation(new BigDecimal(datas[9]).multiply(nwpDirectRadiation).setScale(2, RoundingMode.HALF_UP));//直接辐射
|
|
|
+ aio.setWindSpeed(new BigDecimal(datas[19]).setScale(2, RoundingMode.HALF_UP));
|
|
|
+ aio.setWindDir(new BigDecimal(datas[27]).setScale(2, RoundingMode.HALF_UP));
|
|
|
+ resultList.add(aio);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ jx = false;
|
|
|
+ log.error(station.getStationCode()+" 解析当天NWP文件错误:", e);
|
|
|
+ } finally {
|
|
|
+ close(bufferedReaderNwp, readNwp);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (jx) {
|
|
|
+ resultList.sort(Comparator.comparing(ForecastData::getForecastTime));
|
|
|
+ return resultList;
|
|
|
+ } else {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 关闭文件流
|
|
|
+ *
|
|
|
+ * @param bufferedReader 字符数据
|
|
|
+ * @param read 字节流
|
|
|
+ */
|
|
|
+ private void close(BufferedReader bufferedReader, InputStreamReader read) {
|
|
|
+ try {
|
|
|
+ if (bufferedReader != null) {
|
|
|
+ bufferedReader.close();
|
|
|
+ }
|
|
|
+ if (read != null) {
|
|
|
+ read.close();
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ log.error("关闭文件流失败:", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+}
|