|
|
@@ -1,38 +1,34 @@
|
|
|
package flink.zanxiangnet.ad.monitoring.process;
|
|
|
|
|
|
-import flink.zanxiangnet.ad.monitoring.clickhouse.sink.ClickhouseUtil;
|
|
|
import flink.zanxiangnet.ad.monitoring.pojo.entity.AdStatOfHourDWD;
|
|
|
import flink.zanxiangnet.ad.monitoring.pojo.entity.CostHourDM;
|
|
|
+import flink.zanxiangnet.ad.monitoring.util.DateUtil;
|
|
|
+import org.apache.flink.api.common.state.MapState;
|
|
|
+import org.apache.flink.api.common.state.MapStateDescriptor;
|
|
|
+import org.apache.flink.api.common.typeinfo.Types;
|
|
|
import org.apache.flink.configuration.Configuration;
|
|
|
-import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
|
|
-import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
|
|
|
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
|
|
|
import org.apache.flink.util.Collector;
|
|
|
|
|
|
import java.io.IOException;
|
|
|
-import java.sql.Connection;
|
|
|
-import java.sql.ResultSet;
|
|
|
import java.sql.SQLException;
|
|
|
-import java.sql.Statement;
|
|
|
import java.text.SimpleDateFormat;
|
|
|
-import java.time.format.DateTimeFormatter;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Properties;
|
|
|
+import java.time.LocalDate;
|
|
|
+import java.util.*;
|
|
|
|
|
|
-public class CostHourProcess extends ProcessWindowFunction<AdStatOfHourDWD, CostHourDM, Long, GlobalWindow> {
|
|
|
- private static final DateTimeFormatter formatForLastReduceKey = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH:mm");
|
|
|
- private Connection connection = null;
|
|
|
+public class CostHourProcess extends KeyedProcessFunction<Long, AdStatOfHourDWD, CostHourDM> {
|
|
|
+
|
|
|
+ private MapState<String, Map<Integer, AdStatOfHourDWD>> historyReduceState;
|
|
|
|
|
|
|
|
|
@Override
|
|
|
public void open(Configuration conf) throws SQLException, ClassNotFoundException, IOException {
|
|
|
- Properties props = new Properties();
|
|
|
- props.load(CostHourProcess.class.getResourceAsStream("/application.properties"));
|
|
|
- connection = ClickhouseUtil.getConn(props);
|
|
|
+ historyReduceState = getRuntimeContext().getMapState(new MapStateDescriptor<>("historyReduceState", Types.STRING, Types.MAP(Types.INT, Types.POJO(AdStatOfHourDWD.class))));
|
|
|
}
|
|
|
|
|
|
//数据格式转换
|
|
|
- public CostHourDM datachange(AdStatOfHourDWD adStatOfMinuteDWD, CostHourDM costHourDM) {
|
|
|
+ public CostHourDM dataChange(AdStatOfHourDWD adStatOfMinuteDWD) {
|
|
|
+ CostHourDM costHourDM = new CostHourDM();
|
|
|
//时间-天
|
|
|
costHourDM.dt = adStatOfMinuteDWD.getStatDay();
|
|
|
//计划 id
|
|
|
@@ -202,71 +198,77 @@ public class CostHourProcess extends ProcessWindowFunction<AdStatOfHourDWD, Cost
|
|
|
return costHourDM;
|
|
|
}
|
|
|
|
|
|
-
|
|
|
@Override
|
|
|
- public void process(Long elementCount, ProcessWindowFunction<AdStatOfHourDWD, CostHourDM, Long, GlobalWindow>.Context context,
|
|
|
- Iterable<AdStatOfHourDWD> iterable, Collector<CostHourDM> collector) throws Exception {
|
|
|
-
|
|
|
- //获取前几分钟
|
|
|
-
|
|
|
-
|
|
|
- List<AdStatOfHourDWD> adStatOfMinuteDWDlist = new ArrayList<>(24);
|
|
|
-
|
|
|
- for (AdStatOfHourDWD adStatOfMinuteDWD : iterable) {
|
|
|
- adStatOfMinuteDWDlist.add(adStatOfMinuteDWD);
|
|
|
- CostHourDM costHourDM = new CostHourDM();
|
|
|
-
|
|
|
-
|
|
|
- //当前天
|
|
|
- String statDay = adStatOfMinuteDWD.getStatDay();
|
|
|
- //当前小时
|
|
|
- int hour = adStatOfMinuteDWD.getHour();
|
|
|
- String tmpHour = "";
|
|
|
- int tmpHourInt = adStatOfMinuteDWD.getHour() - 1;
|
|
|
- tmpHour = tmpHourInt > 9 ? String.valueOf(tmpHourInt) : "0" + tmpHourInt;
|
|
|
- String lastHour = adStatOfMinuteDWD.getStatDay() + " " + tmpHour + ":00:00";
|
|
|
-
|
|
|
- tmpHourInt = adStatOfMinuteDWD.getHour() - 2;
|
|
|
- tmpHour = tmpHourInt > 9 ? String.valueOf(tmpHourInt) : "0" + tmpHourInt;
|
|
|
- String lastTwoHour = adStatOfMinuteDWD.getStatDay() + " " + tmpHour + ":00:00";
|
|
|
-
|
|
|
- tmpHourInt = adStatOfMinuteDWD.getHour() - 3;
|
|
|
- tmpHour = tmpHourInt > 9 ? String.valueOf(tmpHourInt) : "0" + tmpHourInt;
|
|
|
- String lastThreeHour = adStatOfMinuteDWD.getStatDay() + " " + tmpHour + ":00:00";
|
|
|
- //TODO:优化为本地拿到小时数据
|
|
|
- String adId = adStatOfMinuteDWD.getAdId().toString();
|
|
|
- String sql = "select " +
|
|
|
- "if(hour='" + lastHour + "',cost_hour,0) last_hour_cost, " +
|
|
|
- "if(hour='" + lastTwoHour + "',cost_hour,0) last_two_hour_cost, " +
|
|
|
- "if(hour='" + lastHour + "',cost_hour,0) - if(hour='" + lastTwoHour + "',cost_hour,0) cost_last_hour_diff, " +
|
|
|
- "(if(hour='" + lastHour + "',cost_hour,0) - if(hour='" + lastTwoHour + "',cost_hour,0))*(if(hour='" + lastTwoHour + "',cost_hour,0) - if(hour='" + lastThreeHour + "',cost_hour,0)) cost_last_three_trend " +
|
|
|
- "from data_monitoring.cost_hour ch " +
|
|
|
- "where dt='" + statDay + "' and ad_id='" + adId + "' ";
|
|
|
+ public void processElement(AdStatOfHourDWD adStatOfHourDWD, KeyedProcessFunction<Long, AdStatOfHourDWD, CostHourDM>.Context context,
|
|
|
+ Collector<CostHourDM> collector) throws Exception {
|
|
|
+ LocalDate day = DateUtil.parseLocalDate(adStatOfHourDWD.getStatDay());
|
|
|
+ Integer hour = adStatOfHourDWD.getHour();
|
|
|
|
|
|
-// System.out.println(sql);
|
|
|
- Statement statement = connection.createStatement();
|
|
|
- ResultSet rs = statement.executeQuery(sql);
|
|
|
- while (rs.next()) {
|
|
|
- costHourDM.costLastHour = rs.getLong(1);
|
|
|
- costHourDM.costLastTwoHour = rs.getLong(2);
|
|
|
- costHourDM.costLastHourDiff = rs.getLong(3);
|
|
|
- costHourDM.costLastThreeTrend = rs.getLong(4);
|
|
|
+ LocalDate lastHourDay = day, lastTwoHourDay = day, lastThreeHourDay = day;
|
|
|
+ int lastHour = hour - 1, lastTwoHour = hour - 2, lastThreeHour = hour - 3;
|
|
|
+ if (hour == 0) {
|
|
|
+ lastHourDay = day.minusDays(1L);
|
|
|
+ lastHour = 23;
|
|
|
+ lastTwoHourDay = day.minusDays(1L);
|
|
|
+ lastTwoHour = 22;
|
|
|
+ lastThreeHourDay = day.minusDays(1L);
|
|
|
+ lastThreeHour = 21;
|
|
|
+ } else if (hour == 1) {
|
|
|
+ lastTwoHourDay = day.minusDays(1L);
|
|
|
+ lastTwoHour = 23;
|
|
|
+ lastThreeHourDay = day.minusDays(1L);
|
|
|
+ lastThreeHour = 22;
|
|
|
+ } else if (hour == 2) {
|
|
|
+ lastThreeHourDay = day.minusDays(1L);
|
|
|
+ lastThreeHour = 23;
|
|
|
+ }
|
|
|
+ long costDiff = 0L, costLastHour = 0L;
|
|
|
+ Map<Integer, AdStatOfHourDWD> lastHourMapping = historyReduceState.get(DateUtil.formatLocalDate(lastHourDay));
|
|
|
+ if (lastHourMapping != null && !lastHourMapping.isEmpty()) {
|
|
|
+ AdStatOfHourDWD lastHourDWD = lastHourMapping.get(lastHour);
|
|
|
+ if (lastHourDWD != null) {
|
|
|
+ costLastHour = lastHourDWD.getCostHour();
|
|
|
+ costDiff = adStatOfHourDWD.getCostHour() - lastHourDWD.getCostHour();
|
|
|
}
|
|
|
+ }
|
|
|
+ long costLastHourDiff = 0, costLastTwoHour = 0;
|
|
|
+ Map<Integer, AdStatOfHourDWD> lastTwoHourMapping = historyReduceState.get(DateUtil.formatLocalDate(lastTwoHourDay));
|
|
|
+ if (lastTwoHourMapping != null && !lastTwoHourMapping.isEmpty()) {
|
|
|
+ AdStatOfHourDWD lastTwoHourDWD = lastTwoHourMapping.get(lastTwoHour);
|
|
|
+ if (lastTwoHourDWD != null) {
|
|
|
+ costLastTwoHour = lastTwoHourDWD.getCostHour();
|
|
|
+ costLastHourDiff = costLastHour - lastTwoHourDWD.getCostHour();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ long costLastTwoHourDiff = 0, costLastThreeTrend = 0;
|
|
|
+ Map<Integer, AdStatOfHourDWD> lastThreeHourMapping = historyReduceState.get(DateUtil.formatLocalDate(lastThreeHourDay));
|
|
|
+ if (lastThreeHourMapping != null && !lastThreeHourMapping.isEmpty()) {
|
|
|
+ AdStatOfHourDWD lastThreeHourDWD = lastThreeHourMapping.get(lastThreeHour);
|
|
|
+ if (lastThreeHourDWD != null) {
|
|
|
+ costLastThreeTrend = lastThreeHourDWD.getCostHour();
|
|
|
+ costLastTwoHourDiff = costLastTwoHour - lastThreeHourDWD.getCostHour();
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- CostHourDM costHourDM_new = datachange(adStatOfMinuteDWD, costHourDM);
|
|
|
-
|
|
|
- collector.collect(costHourDM_new);
|
|
|
-// System.out.println("costhour_输入" + JsonUtil.toString(adStatOfMinuteDWD));
|
|
|
-// System.out.println("costhour_输出：" + JsonUtil.toString(costHourDM_new));
|
|
|
+ if (historyReduceState.get(adStatOfHourDWD.getStatDay()) == null) {
|
|
|
+ Map<Integer, AdStatOfHourDWD> hourMapping = new HashMap<>(24);
|
|
|
+ hourMapping.put(adStatOfHourDWD.getHour(), adStatOfHourDWD);
|
|
|
+ historyReduceState.put(adStatOfHourDWD.getStatDay(), hourMapping);
|
|
|
+ } else {
|
|
|
+ historyReduceState.get(adStatOfHourDWD.getStatDay()).put(adStatOfHourDWD.getHour(), adStatOfHourDWD);
|
|
|
}
|
|
|
-// System.out.println("costhour_windowCount：" + adStatOfMinuteDWDlist.size());
|
|
|
|
|
|
+ CostHourDM costHourDM = dataChange(adStatOfHourDWD);
|
|
|
+ costHourDM.setCostDiff(costDiff);
|
|
|
+ costHourDM.setCostLastHour(costLastHour);
|
|
|
+ costHourDM.setCostLastHourDiff(costLastHourDiff);
|
|
|
+ costHourDM.setCostLastTwoHour(costLastTwoHour);
|
|
|
+ costHourDM.setCostLastTwoHourDiff(costLastTwoHourDiff);
|
|
|
+ costHourDM.setCostLastThreeTrend(costLastThreeTrend);
|
|
|
|
|
|
- }
|
|
|
+ collector.collect(costHourDM);
|
|
|
+ historyReduceState.remove(DateUtil.formatLocalDate(day.minusDays(2L)));
|
|
|
+ historyReduceState.remove(DateUtil.formatLocalDate(day.minusDays(3L)));
|
|
|
|
|
|
- @Override
|
|
|
- public void clear(ProcessWindowFunction<AdStatOfHourDWD, CostHourDM, Long, GlobalWindow>.Context context) throws Exception {
|
|
|
- System.out.println("窗口关闭");
|
|
|
}
|
|
|
-
|
|
|
}
|
