package com.jiayue.biz.service.impl; import com.jiayue.biz.domain.FakeDataRecord; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; import org.joda.time.Minutes; import java.lang.reflect.Field; import java.math.BigDecimal; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; /** * 为审核数据的合理性提供历史数据 * * @author 修唯xiuwei * @version 3.0 */ @Data @Slf4j public class CheckDataRecode { /*** * 原始值 */ private BigDecimal originalVal; /*** * 使用值 */ private BigDecimal usedVal; /** * 死值开始时间 */ private DateTime deadDataStartTime = DateTime.now(); /** * 该数据的获取时间 */ private DateTime time; public static Map map = new HashMap<>(); public List> checkValue(List> allData, String type) { List> filterData = new ArrayList<>(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); if (type.equals("rld") || type.equals("station") || type.equals("dat")) { simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); } if (type.equals("sld")) { simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmm"); } try { for (Map map : allData) { StringBuilder abnormalType = new StringBuilder(); Long date; try { date = simpleDateFormat.parse(map.get("time").toString()).getTime(); } catch (ParseException e) { date = Long.parseLong(map.get("time").toString()); } Long time = date; map.forEach((key, value) -> { //风速 if (key.contains("ws")) { int max; int min; int change; if (key.contains("Sta") || key.contains("sta")) { max = 6; min = 0; change = 1; } else { max = 40; min = 0; change = 20; } if (!check(value, max, min, change, key, time).equals("")) { if (!abnormalType.toString().equals("")) { abnormalType.append(","); } abnormalType.append(check(value, max, min, change, key, time)); } } //湿度 if (key.contains("rh")) { int max; int min; int change; if (key.contains("Sta") || key.contains("sta")) { max = 6; min = 0; change = 1; } else { max = 100; min = 0; change = 40; } if (!check(value, max, min, change, key, time).equals("")) { if (!abnormalType.toString().equals("")) { abnormalType.append(","); } abnormalType.append(check(value, max, min, change, key, time)); } } //温度 if (key.equals("tMax") || key.equals("tAve") || key.equals("tInst") || key.equals("tSta") || key.equals("t_max") || key.equals("t_ave") || key.equals("t_inst") || key.equals("t_sta")) { int max; int min; int change; if (key.contains("Sta") || key.contains("sta")) { max = 6; min = 0; change = 1; } else { max = 50; min = -50; change = 10; } if (!check(value, max, min, change, key, time).equals("")) { if (!abnormalType.toString().equals("")) { abnormalType.append(","); } abnormalType.append(check(value, max, min, change, key, time)); } } //气压 if (key.contains("pa") && !key.equals("parent_id")) { int max; int min; int change; if (key.contains("Sta") || key.contains("sta")) { max = 6; min = 0; change = 1; } else { max = 1200; min = 800; change = 5; } //气压低于200被认为是kpa 转hpa if (new BigDecimal(value.toString()).compareTo(new BigDecimal(200)) < 0) { value = new BigDecimal(value.toString()).multiply(new BigDecimal(10)).toString(); } if (!check(value, max, min, change, key, time).equals("")) { if (!abnormalType.toString().equals("")) { abnormalType.append(","); } abnormalType.append(check(value, max, min, change, key, time)); } } }); if (!abnormalType.toString().equals("")) { map.put("abnormalData", "1"); map.put("abnormalType", abnormalType.toString()); } filterData.add(map); } Map startTimeAndEndTimeMap = map.entrySet().stream().filter(k -> k.getKey().contains("DeadDataStartTimeAndEndTime")).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); compareData(startTimeAndEndTimeMap, filterData,simpleDateFormat); } catch (Exception e) { log.error("数据筛选异常" + e); e.printStackTrace(); } return filterData; } public void compareData(Map startTimeAndEndTimeMap, List> filterData,SimpleDateFormat simpleDateFormat) { for (Map filterDatum : filterData) { for (Map.Entry entry : startTimeAndEndTimeMap.entrySet()) { //获取异常数据字段名 String dataName = entry.getKey().replace("DeadDataStartTimeAndEndTime", ""); //是否存在多时间段异常数据 if (entry.getValue().toString().contains(",")) { String[] startTimeAndEndTime = entry.getValue().toString().split(","); for (String time : startTimeAndEndTime) { //保存异常值 replaceData(time, filterDatum, dataName, simpleDateFormat); } } else { replaceData(entry.getValue().toString(), filterDatum, dataName, simpleDateFormat); } } } } public void replaceData(String value, Map filterDatum, String dataName,SimpleDateFormat simpleDateFormat){ Long time; try { time = simpleDateFormat.parse(filterDatum.get("time").toString()).getTime(); } catch (ParseException e) { time = Long.parseLong(filterDatum.get("time").toString()); } String[] startTimeAndEndTimeSp = value.split("-"); //在时间段内 并且没有此字段名 if (time >= Long.parseLong(startTimeAndEndTimeSp[0]) && time <= Long.parseLong(startTimeAndEndTimeSp[1])) { if (filterDatum.get("abnormalType") != null && !filterDatum.get("abnormalType").toString().contains(dataName)) { filterDatum.put("abnormalType", filterDatum.get("abnormalType") + "," + dataName); } } } public String check(Object object, Integer max, Integer min, Integer change, String name, Long time) { String s = ""; if (null != object) { BigDecimal val = new BigDecimal(object.toString()); initValue(val, max, min, name, time); FakeDataRecord.IrrationalDataType audit = isRational(val, max, min, change, name, time); //当数据合理时 if (audit == FakeDataRecord.IrrationalDataType.Rational) { map.put(name + "Time", DateTime.now()); map.put(name + "OriginalVal", val); map.put(name + "UsedVal", val); } else { s = name; } } return s; } public void initValue(BigDecimal val, Integer max, Integer min, String name, Long time) { if (map.get(name + "UsedVal") == null) { if (val.doubleValue() > max) { map.put(name + "UsedVal", BigDecimal.valueOf(max)); } else if (val.doubleValue() < min) { map.put(name + "UsedVal", BigDecimal.valueOf(min)); } else { map.put(name + "UsedVal", val); } } map.putIfAbsent(name + "OriginalVal", val); map.putIfAbsent(name + "Time", new DateTime(time)); map.putIfAbsent(name + "DeadDataStartTime", new DateTime(time)); } /** * 判断数据是否合理 * * @param value * @return */ public FakeDataRecord.IrrationalDataType isRational(BigDecimal value, Integer max, Integer min, Integer change, String name, Long time) { //平均值在正常情况下 风速最大值没超过35 即使最大值已经被冻结 依然不标记成异常数据 //风速 if (name.contains("ws")) { //最大值和平均值需要筛选 if (name.contains("Max") || name.contains("Ave") || name.contains("max") || name.contains("ave")) { if (isOverMax(name, value, max)) { return FakeDataRecord.IrrationalDataType.OverMax; } if (isLowerMin(name, value, min)) { return FakeDataRecord.IrrationalDataType.LowerMin; } if (isDeadVal(name, value, time)) { return FakeDataRecord.IrrationalDataType.DeadVal; } if (isSuddenChange(name, value, change)) { return FakeDataRecord.IrrationalDataType.SuddenChange; } } else { return FakeDataRecord.IrrationalDataType.Rational; } } else { if (isOverMax(name, value, max)) { return FakeDataRecord.IrrationalDataType.OverMax; } if (isLowerMin(name, value, min)) { return FakeDataRecord.IrrationalDataType.LowerMin; } if (isDeadVal(name, value, time)) { return FakeDataRecord.IrrationalDataType.DeadVal; } if (isSuddenChange(name, value, change)) { return FakeDataRecord.IrrationalDataType.SuddenChange; } } return FakeDataRecord.IrrationalDataType.Rational; } /** * 数据是否超上限 * * @param value 传入值 * @return 是否超上限 */ private static boolean isOverMax(String name, BigDecimal value, Integer max) { if (BigDecimal.valueOf(max).compareTo(value) < 0) { log.warn("{}数据超过上限,数值为{},上限为{}", name, value, max); return true; } else { return false; } } /** * 数据是否越下限 * * @param value 传入值 * @return 是否越下限 */ private static boolean isLowerMin(String name, BigDecimal value, Integer min) { if (BigDecimal.valueOf(min).compareTo(value) > 0) { log.warn("{}数据超过下限,数值为{},下限为{}", name, value, min); return true; } else { return false; } } /** * 数据是否是死数据 * * @param value 传入值 * @return 是否是死值 */ private boolean isDeadVal(String name, BigDecimal value, Long time) { DateTime dateTime = new DateTime(time); boolean flag = false; //上个合理值不等于空并且上个合理值和传入值相等 if (map.get(name + "OriginalVal") != null && map.get(name + "OriginalVal").equals(value)) { String ddst = map.get(name + "DeadDataStartTime").toString(); //如果死值数据开始时间不为空 并且 死值开始时间和传入的时间差距大于30分钟 if (!"".equals(ddst) && Minutes.minutesBetween(new DateTime(ddst), dateTime).getMinutes() > 30) { log.warn("{}数据死值,数值为{},开始时间为{}", name, value, new DateTime(ddst).toString("HH:mm:ss")); flag = true; } } else { map.put(name + "DeadDataStartTime", dateTime); } return flag; } /** * 数据是否为突变数据 * * @param value 传入值 * @return boolean 是否突变 */ private boolean isSuddenChange(String name, BigDecimal value, Integer change) { if (value.subtract(new BigDecimal(map.get(name + "UsedVal").toString())).abs().compareTo(BigDecimal.valueOf(change)) == 1) { log.warn("{}数据超过突变极限,数值为{},上一个值为{},突变极限为{}", name, value, map.get(name + "UsedVal"), change); return true; } else { return false; } } }