Merge branch 'master' of https://git.zanxiangnet.com/cxyu/DataMonitoring

flink-ad-monitoring/dependency-reduced-pom.xml

@@ -88,7 +88,7 @@
               </filters>
               <transformers>
                 <transformer>
-                  <mainClass>flink.zanxiangnet.ad.monitoring.AdStatJob</mainClass>
+                  <mainClass>flink.zanxiangnet.ad.monitoring.PlanHourStreamJob</mainClass>
                 </transformer>
               </transformers>
             </configuration>

flink-ad-monitoring/pom.xml

@@ -118,6 +118,12 @@ under the License.
             <version>${odps.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>3.10.2</version>
+        </dependency>
+
         <dependency>
             <groupId>com.tencent.ads</groupId>
             <artifactId>marketing-api-java-sdk</artifactId>
@@ -145,9 +151,9 @@ under the License.
 
         <!-- Java操作 csv文件 -->
         <dependency>
-            <groupId>net.sf.supercsv</groupId>
-            <artifactId>super-csv</artifactId>
-            <version>2.4.0</version>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.9.0</version>
         </dependency>
     </dependencies>
 
@@ -202,7 +208,7 @@ under the License.
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>flink.zanxiangnet.ad.monitoring.AdStatJob</mainClass>
+                                    <mainClass>flink.zanxiangnet.ad.monitoring.PlanHourStreamJob</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/AdStatJob.java

@@ -85,54 +85,23 @@ public class AdStatJob {
         // 广告小时数据（往前回滚 10天）
         final OutputTag<AdDataOfHourODS> adHourStreamTag = new OutputTag<AdDataOfHourODS>("adHourStream") {
         };
+        // 广告小时数据（从分钟数据流切出来的整点数据）
+        final OutputTag<AdStatOfHourDWD> adHourFromMinuteStreamTag = new OutputTag<AdStatOfHourDWD>("adHourFromMinuteStream") {
+        };
 
         // 对流进行映射，拆分（实时的分钟流和回滚的小时流）
         SingleOutputStreamOperator<AdDataOfMinuteODS> adODSStream = adStreamOfMinuteIn.filter(StringUtils::isNotBlank)
-                .process(new ProcessFunction<String, AdDataOfMinuteODS>() {
-                    @Override
-                    public void processElement(String adDataStr, ProcessFunction<String, AdDataOfMinuteODS>.Context context, Collector<AdDataOfMinuteODS> collector) throws Exception {
-                        AdDataOfMinuteDTO dto = JsonUtil.toObj(adDataStr, AdDataOfMinuteDTO.class);
-                        // 指记录被创建的时间
-                        long createTime = dto.getCreateTime();
-                        // 指记录统计的时间（如果是实时统计的数据精确到分钟，回滚的历史数据精确到天）
-                        long statTime = DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(dto.getDataTime()).withSecond(0).withNano(0));
-                        HourlyReportsGetListStruct struct = dto.getHourlyReportsGetListStruct();
-
-                        if (createTime - statTime > (60 * 60 * 1000L)) {
-                            AdDataOfHourODS adODS = new AdDataOfHourODS();
-                            BeanUtils.copyProperties(struct, adODS);
-                            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
-                            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
-                            adODS.setStatTime(DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(statTime).withMinute(0)));
-                            adODS.setAccountId(dto.getAccountId());
-                            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
-                            adODS.setCreateTime(new Date(createTime));
-                            adODS.removeNull();
-                            context.output(adHourStreamTag, adODS);
-                        } else {
-                            AdDataOfMinuteODS adODS = new AdDataOfMinuteODS();
-                            BeanUtils.copyProperties(struct, adODS);
-                            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
-                            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
-                            adODS.setStatTime(statTime);
-                            adODS.setAccountId(dto.getAccountId());
-                            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
-                            adODS.setCreateTime(new Date(createTime));
-                            adODS.removeNull();
-                            context.output(adMinuteStreamTag, adODS);
-                        }
-                    }
-                });
+                .process(new AdHourDTOStreamProcess(adMinuteStreamTag, adHourStreamTag));
 
         // 分钟流
         DataStream<AdDataOfMinuteODS> adMinuteODSStream = adODSStream.getSideOutput(adMinuteStreamTag);
-        // 写入原始表
-        // adMinuteODSStream.addSink(new TunnelBatchSink<>(AdDataOfMinuteODS.class, 36000L, 64000L, 3));
+        // 分钟流-写入原始表
         new KeyedBatchStream<>("adMinuteODSStream", adMinuteODSStream.keyBy(AdDataOfMinuteODS::getStatDay), 4000L, 2 * 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdDataOfMinuteODS.class))
                 .name("sink_ad_minute_ods");
 
+        // 分钟流-计算
         SingleOutputStreamOperator<AdStatOfMinuteDWD> adMinuteDWDStream = adMinuteODSStream
                 // 打水印，允许数据延迟 6分钟，同时指定时间流
                 .assignTimestampsAndWatermarks(WatermarkStrategy.<AdDataOfMinuteODS>forBoundedOutOfOrderness(Duration.ofMinutes(6L))
@@ -141,8 +110,7 @@ public class AdStatJob {
                 // 开一个 5分钟的滚动窗口
                 .window(TumblingEventTimeWindows.of(Time.minutes(5L)))
                 .trigger(new AdMinuteODSStreamTrigger())
-                .process(new AdMinuteDWDProcess());
-        // .addSink(new TunnelBatchSink<>(AdStatOfMinuteDWD.class, 30000L, 365L, 6));
+                .process(new AdMinuteDWDProcess(adHourFromMinuteStreamTag));
         new KeyedBatchStream<>("adMinuteDWDStream", adMinuteDWDStream.keyBy(AdStatOfMinuteDWD::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdStatOfMinuteDWD.class))
@@ -162,7 +130,7 @@ public class AdStatJob {
 
 
         //cost----小时数据处理
-        SingleOutputStreamOperator<CostHourDM> clickhouseMinuteHourDmStream =
+        /*SingleOutputStreamOperator<CostHourDM> clickhouseMinuteHourDmStream =
                 adMinuteDWDStream
                         .keyBy(AdStatOfMinuteDWD::getAdId)
                         .window(TumblingEventTimeWindows.of(Time.minutes(5L)))
@@ -171,92 +139,31 @@ public class AdStatJob {
                         .name("sink_ad_minute_hour_dm_clickhouse");
 
         BatchSinkHour batchSinkMinuteHour = new BatchSinkHour();
-        clickhouseMinuteHourDmStream.addSink(batchSinkMinuteHour);
+        clickhouseMinuteHourDmStream.addSink(batchSinkMinuteHour);*/
 
 
-        // 小时流（直接写到小时报表的 ods）
+        // 小时流
         DataStream<AdDataOfHourODS> adHourODSStream = adODSStream.getSideOutput(adHourStreamTag);
-        // 写入原始表
-        // adHourODSStream.addSink(new TunnelBatchSink<>(AdDataOfHourODS.class, 36000L, 64000L, 10));
+        // 小时流-写入原始表
         new KeyedBatchStream<>("adHourODSStream", adHourODSStream.keyBy(AdDataOfHourODS::getStatDay), 4000L, 5 * 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdDataOfHourODS.class))
                 .name("sink_ad_hour_ods");
 
+        // 小时流-计算
         SingleOutputStreamOperator<AdStatOfHourDWD> adHourDWDStream =
                 adHourODSStream.keyBy(AdDataOfHourODS::getAdId)
-                        .countWindow(1).
-                        process(new ProcessWindowFunction<AdDataOfHourODS, AdStatOfHourDWD, Long, GlobalWindow>() {
-                            private Odps odps;
-                            // 上次查询的天数据
-                            private ValueState<String> lastQueryDayState;
-                            // 聚合的天的数据
-                            private MapState<String, AdStatOfHourDWD> historyReduceState;
-
-                            @Override
-                            public void open(Configuration conf) {
-                                Map<String, String> params = getRuntimeContext()
-                                        .getExecutionConfig()
-                                        .getGlobalJobParameters()
-                                        .toMap();
-                                Account account = new AliyunAccount(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ID),
-                                        params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_KEY));
-                                odps = new Odps(account);
-                                odps.getRestClient().setRetryLogger(new MaxComputeLog());
-                                odps.setEndpoint(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ENDPOINT));
-                                odps.setDefaultProject(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_PROJECT_NAME));
-
-                                lastQueryDayState = getRuntimeContext().getState(new ValueStateDescriptor<>("lastQueryDayState", String.class));
-                                historyReduceState = getRuntimeContext().getMapState(new MapStateDescriptor<>("historyReduceState", Types.STRING, Types.POJO(AdStatOfHourDWD.class)));
-                            }
-
-                            @Override
-                            public void process(Long elementsCount, ProcessWindowFunction<AdDataOfHourODS, AdStatOfHourDWD, Long, GlobalWindow>.Context context, Iterable<AdDataOfHourODS> iterable, Collector<AdStatOfHourDWD> collector) throws Exception {
-                                AdDataOfHourODS element = iterable.iterator().next();
-                                LocalDate statDay = DateUtil.parseLocalDate(element.getStatDay());
-                                LocalDateTime statTime = LocalDateTime.of(statDay, LocalTime.of(element.getHour(), 0, 0));
-                                long now = System.currentTimeMillis();
-                                LocalDate today = LocalDate.now();
-
-                                String lastQueryDay = lastQueryDayState.value();
-                                // 从 maxCompute拉取指定 广告的历史数据
-                                if (lastQueryDay == null || !lastQueryDay.equals(DateUtil.formatLocalDate(today))) {
-                                    LocalDate endDay = today, beginDay = statDay.minusDays(60);
-                                    String sql = "SELECT * FROM ad_stat_of_hour_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endDay) + "\" AND ad_id = " + element.getAdId() + ";";
-                                    Instance instance = SQLTask.run(odps, sql);
-                                    System.out.println("212===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
-                                    instance.waitForSuccess();
-                                    List<Record> records = SQLTask.getResult(instance);
-                                    Map<String, AdStatOfHourDWD> historyHourMap = records.stream()
-                                            .map(AdStatOfHourDWD::byMaxCompute)
-                                            .sorted((o1, o2) -> new Long(o1.getCreateTime().getTime() - o2.getCreateTime().getTime()).intValue())
-                                            .collect(Collectors.toMap(data -> data.getStatDay() + data.getHour(), data -> data, (val1, val2) -> val2));
-                                    historyReduceState.clear();
-                                    historyReduceState.putAll(historyHourMap);
-                                    lastQueryDayState.update(DateUtil.formatLocalDate(today));
-                                }
-                                AdStatOfHourDWD lastReduceData = null;
-                                for (int i = 1; i < 60 * 24; i++) {
-                                    LocalDateTime time = statTime.minusHours(i);
-                                    lastReduceData = historyReduceState.get(DateUtil.formatLocalDate(time.toLocalDate()) + time.getHour());
-                                    if (lastReduceData != null) {
-                                        break;
-                                    }
-                                }
-
-                                AdStatOfHourDWD newStatData = AdStatOfHourDWD.reduce(lastReduceData, element, now);
-                                collector.collect(newStatData);
-                            }
-                        });
-        //.addSink(new TunnelBatchSink<>(AdStatOfHourDWD.class, 30000L, 365L, 6));
-        new KeyedBatchStream<>("adHourDWDStream", adHourDWDStream.keyBy(AdStatOfHourDWD::getStatDay), 4000L, 60 * 1000L)
+                        .countWindow(1)
+                        .process(new AdHourDWDProcess());
+        DataStream<AdStatOfHourDWD> adHourDWDAllStream = adMinuteDWDStream.getSideOutput(adHourFromMinuteStreamTag).union(adHourDWDStream);
+        new KeyedBatchStream<>("adHourDWDStream", adHourDWDAllStream.keyBy(AdStatOfHourDWD::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdStatOfHourDWD.class))
                 .name("sink_ad_hour_dwd");
 
         //小时数据
         SingleOutputStreamOperator<CostHourDM> clickhouseHourDmStream =
-                adHourDWDStream
+                adHourDWDAllStream
                         .keyBy(AdStatOfHourDWD::getAdId)
                         .countWindow(1)
                         .process(new CostHourProcess())
@@ -300,7 +207,6 @@ public class AdStatJob {
                             .adDataOfDayODS(adODS)
                             .build();
                 });
-        // adDayODSStream.map(AdStatOfDayODSDTO::getAdDataOfDayODS).addSink(new TunnelBatchSink<>(AdDataOfDayODS.class, 36000L, 6000L, 10));
         new KeyedBatchStream<>("adDayODSStream", adDayODSStream.map(AdStatOfDayODSDTO::getAdDataOfDayODS).keyBy(AdDataOfDayODS::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdDataOfDayODS.class))
@@ -361,7 +267,7 @@ public class AdStatJob {
                             LocalDate endTime = LocalDate.now(), beginTime = statDay.minusDays(60);
                             String sql = "SELECT * FROM ad_stat_of_day_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginTime) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endTime) + "\" AND ad_id = " + element.getAdId() + ";";
                             Instance instance = SQLTask.run(odps, sql);
-                            System.out.println("337===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+                            // log.error("sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
                             instance.waitForSuccess();
                             List<Record> records = SQLTask.getResult(instance);
                             Map<String, AdStatOfDayDWD> historyData = records.stream()
@@ -422,7 +328,6 @@ public class AdStatJob {
                         out.collect(newStatDWD);
                     }
                 });
-        //.addSink(new TunnelBatchSink<>(AdStatOfDayDWD.class, 30000L, 365L, 6));
         new KeyedBatchStream<>("adDayDWDYearStream", adDayDWDYearStream.keyBy(AdStatOfDayDWD::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(AdStatOfDayDWD.class))

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/PlanStatJob.java → flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/PlanDayStreamJob.java

@@ -7,28 +7,24 @@ import com.aliyun.odps.account.AliyunAccount;
 import com.aliyun.odps.data.Record;
 import com.aliyun.odps.task.SQLTask;
 import com.tencent.ads.model.DailyReportsGetListStruct;
-import com.tencent.ads.model.HourlyReportsGetListStruct;
+import flink.zanxiangnet.ad.monitoring.kafka.KafkaComponent;
 import flink.zanxiangnet.ad.monitoring.maxcompute.MaxComputeLog;
 import flink.zanxiangnet.ad.monitoring.pojo.dto.AdDataOfDayDTO;
-import flink.zanxiangnet.ad.monitoring.pojo.dto.AdDataOfMinuteDTO;
 import flink.zanxiangnet.ad.monitoring.pojo.dto.AdStatOfDayODSDTO;
-import flink.zanxiangnet.ad.monitoring.pojo.entity.*;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfDayODS;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.PlanStatOfDayDWD;
 import flink.zanxiangnet.ad.monitoring.pojo.properties.ApplicationProperties;
-import flink.zanxiangnet.ad.monitoring.process.PlanMinuteDWDProcess;
 import flink.zanxiangnet.ad.monitoring.sink.TunnelBatchStreamSink;
 import flink.zanxiangnet.ad.monitoring.stream.KeyedBatchStream;
-import flink.zanxiangnet.ad.monitoring.trigger.PlanMinuteODSStreamTrigger;
 import flink.zanxiangnet.ad.monitoring.util.DateUtil;
 import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
-import flink.zanxiangnet.ad.monitoring.kafka.KafkaComponent;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.state.MapState;
 import org.apache.flink.api.common.state.MapStateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeinfo.Types;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connector.kafka.source.KafkaSource;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
@@ -36,176 +32,32 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.ProcessFunction;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.OutputTag;
 import org.springframework.beans.BeanUtils;
 
-import java.time.Duration;
 import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.*;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.stream.Collectors;
 
-public class PlanStatJob {
-
-    /**
-     * 可能有数据的最早日期
-     */
-    private static final String OLDEST_DAY = "2019-01-01";
-
+@Slf4j
+public class PlanDayStreamJob {
     public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
         // 加载配置文件到 flink的全局配置中
         Properties props = new Properties();
-        props.load(PlanStatJob.class.getResourceAsStream("/application.properties"));
+        props.load(PlanHourStreamJob.class.getResourceAsStream("/application.properties"));
         Configuration configuration = new Configuration();
         props.stringPropertyNames().forEach(key -> {
             String value = props.getProperty(key);
             configuration.setString(key.trim(), StringUtils.isBlank(value) ? "" : value.trim());
         });
         env.getConfig().setGlobalJobParameters(configuration);
-
-        KafkaSource<String> adStreamOfMinuteSource = KafkaComponent.buildKafkaSource(props, KafkaComponent.KafkaTopic.adHourTopic, KafkaComponent.KafkaTopic.KafkaGroupId.planHourConsumerGroup);
-        DataStreamSource<String> adStreamOfMinuteIn = env.fromSource(adStreamOfMinuteSource, WatermarkStrategy.noWatermarks(), "planHourSource_kafka");
-
-        // 广告分钟数据（前 5分钟的广告消耗数据）
-        final OutputTag<AdDataOfMinuteODS> adMinuteStreamTag = new OutputTag<AdDataOfMinuteODS>("adMinuteStream") {
-        };
-        // 广告小时数据（往前回滚 10天）
-        final OutputTag<AdDataOfHourODS> adHourStreamTag = new OutputTag<AdDataOfHourODS>("adHourStream") {
-        };
-
-        // 对流进行映射，拆分（实时的分钟流和回滚的小时流）
-        SingleOutputStreamOperator<AdDataOfMinuteODS> adODSStream = adStreamOfMinuteIn.filter(StringUtils::isNotBlank)
-                .process(new ProcessFunction<String, AdDataOfMinuteODS>() {
-                    @Override
-                    public void processElement(String adDataStr, ProcessFunction<String, AdDataOfMinuteODS>.Context context, Collector<AdDataOfMinuteODS> collector) throws Exception {
-                        AdDataOfMinuteDTO dto = JsonUtil.toObj(adDataStr, AdDataOfMinuteDTO.class);
-                        // 指记录被创建的时间
-                        long createTime = dto.getCreateTime();
-                        // 指记录统计的时间（如果是实时统计的数据精确到分钟，回滚的历史数据精确到天）
-                        long statTime = DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(dto.getDataTime()).withSecond(0).withNano(0));
-                        HourlyReportsGetListStruct struct = dto.getHourlyReportsGetListStruct();
-
-                        if (createTime - statTime > (60 * 60 * 1000L)) {
-                            AdDataOfHourODS adODS = new AdDataOfHourODS();
-                            BeanUtils.copyProperties(struct, adODS);
-                            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
-                            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
-                            adODS.setStatTime(DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(statTime).withMinute(0)));
-                            adODS.setAccountId(dto.getAccountId());
-                            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
-                            adODS.setCreateTime(new Date(createTime));
-                            adODS.removeNull();
-                            context.output(adHourStreamTag, adODS);
-                        } else {
-                            AdDataOfMinuteODS adODS = new AdDataOfMinuteODS();
-                            BeanUtils.copyProperties(struct, adODS);
-                            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
-                            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
-                            adODS.setStatTime(statTime);
-                            adODS.setAccountId(dto.getAccountId());
-                            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
-                            adODS.setCreateTime(new Date(createTime));
-                            adODS.removeNull();
-                            context.output(adMinuteStreamTag, adODS);
-                        }
-                    }
-                });
-
-        // 分钟流
-        SingleOutputStreamOperator<PlanStatOfMinuteDWD> planMinuteDWDStream = adODSStream.getSideOutput(adMinuteStreamTag)
-                // 打水印，允许数据延迟 6分钟，同时指定时间流
-                .assignTimestampsAndWatermarks(WatermarkStrategy.<AdDataOfMinuteODS>forBoundedOutOfOrderness(Duration.ofMinutes(6L))
-                        .withTimestampAssigner((SerializableTimestampAssigner<AdDataOfMinuteODS>) (adODS, l) -> adODS.getStatTime()))
-                .keyBy(AdDataOfMinuteODS::getCampaignId)
-                // 开一个 5分钟的滚动窗口
-                .window(TumblingEventTimeWindows.of(Time.minutes(5L)))
-                .trigger(new PlanMinuteODSStreamTrigger())
-                .process(new PlanMinuteDWDProcess());
-        // .addSink(new TunnelBatchSink<>(AdStatOfMinuteDWD.class, 30000L, 365L, 6));
-        new KeyedBatchStream<>("planMinuteDWDStream", planMinuteDWDStream.keyBy(PlanStatOfMinuteDWD::getStatDay), 4000L, 60 * 1000L)
-                .toBatch()
-                .addSink(new TunnelBatchStreamSink<>(PlanStatOfMinuteDWD.class))
-                .name("sink_plan_minute_dwd");
-
-        // 小时流
-        SingleOutputStreamOperator<PlanStatOfHourDWD> planHourDWDStream = adODSStream.getSideOutput(adHourStreamTag).keyBy(AdDataOfHourODS::getCampaignId)
-                .countWindow(1).process(new ProcessWindowFunction<AdDataOfHourODS, PlanStatOfHourDWD, Long, GlobalWindow>() {
-                    private Odps odps;
-                    // 上次查询的天数据
-                    private ValueState<String> lastQueryDayState;
-                    // 聚合的天的数据
-                    private MapState<String, PlanStatOfHourDWD> historyReduceState;
-
-                    @Override
-                    public void open(Configuration conf) {
-                        Map<String, String> params = getRuntimeContext()
-                                .getExecutionConfig()
-                                .getGlobalJobParameters()
-                                .toMap();
-                        Account account = new AliyunAccount(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ID),
-                                params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_KEY));
-                        odps = new Odps(account);
-                        odps.getRestClient().setRetryLogger(new MaxComputeLog());
-                        odps.setEndpoint(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ENDPOINT));
-                        odps.setDefaultProject(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_PROJECT_NAME));
-
-                        lastQueryDayState = getRuntimeContext().getState(new ValueStateDescriptor<>("lastQueryDayState", String.class));
-                        historyReduceState = getRuntimeContext().getMapState(new MapStateDescriptor<>("historyReduceState", Types.STRING, Types.POJO(PlanStatOfHourDWD.class)));
-                    }
-
-                    @Override
-                    public void process(Long elementsCount, ProcessWindowFunction<AdDataOfHourODS, PlanStatOfHourDWD, Long, GlobalWindow>.Context context,
-                                        Iterable<AdDataOfHourODS> iterable, Collector<PlanStatOfHourDWD> collector) throws Exception {
-                        AdDataOfHourODS element = iterable.iterator().next();
-                        LocalDate statDay = DateUtil.parseLocalDate(element.getStatDay());
-                        LocalDateTime statTime = LocalDateTime.of(statDay, LocalTime.of(element.getHour(), 0, 0));
-                        long now = System.currentTimeMillis();
-                        LocalDate today = LocalDate.now();
-
-                        String lastQueryDay = lastQueryDayState.value();
-                        // 从 maxCompute拉取指定 广告的历史数据
-                        if (lastQueryDay == null || !lastQueryDay.equals(DateUtil.formatLocalDate(today))) {
-                            LocalDate endDay = today, beginDay = statDay.minusDays(60);
-                            String sql = "SELECT * FROM plan_stat_of_hour_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endDay) + "\" AND campaign_id = " + element.getCampaignId() + ";";
-                            Instance instance = SQLTask.run(odps, sql);
-                            System.out.println("178===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
-                            instance.waitForSuccess();
-                            List<Record> records = SQLTask.getResult(instance);
-                            Map<String, PlanStatOfHourDWD> historyHourMap = records.stream()
-                                    .map(PlanStatOfHourDWD::byMaxCompute)
-                                    .sorted((o1, o2) -> new Long(o1.getCreateTime().getTime() - o2.getCreateTime().getTime()).intValue())
-                                    .collect(Collectors.toMap(data -> data.getStatDay() + data.getHour(), data -> data, (val1, val2) -> val2));
-                            historyReduceState.clear();
-                            historyReduceState.putAll(historyHourMap);
-                            lastQueryDayState.update(DateUtil.formatLocalDate(today));
-                        }
-                        PlanStatOfHourDWD lastReduceData = null;
-                        for (int i = 1; i < 60 * 24; i++) {
-                            LocalDateTime time = statTime.minusHours(i);
-                            lastReduceData = historyReduceState.get(DateUtil.formatLocalDate(time.toLocalDate()) + time.getHour());
-                            if (lastReduceData != null) {
-                                break;
-                            }
-                        }
-
-                        PlanStatOfHourDWD newStatData = PlanStatOfHourDWD.reduce(lastReduceData, element, now);
-                        collector.collect(newStatData);
-                    }
-                });
-        //.addSink(new TunnelBatchSink<>(AdStatOfHourDWD.class, 30000L, 365L, 6));
-        new KeyedBatchStream<>("planHourDWDStream", planHourDWDStream.keyBy(PlanStatOfHourDWD::getStatDay), 4000L, 60 * 1000L)
-                .toBatch()
-                .addSink(new TunnelBatchStreamSink<>(PlanStatOfHourDWD.class))
-                .name("sink_plan_hour_dwd");
-
-        // ------------------------------------------------------- 处理广告的天数据 -----------------------------------------
         KafkaSource<String> adStreamOfDaySource = KafkaComponent.buildKafkaSource(props, KafkaComponent.KafkaTopic.adDayTopic, KafkaComponent.KafkaTopic.KafkaGroupId.planDayConsumerGroup);
 
         DataStreamSource<String> adStreamOfDayIn = env.fromSource(adStreamOfDaySource, WatermarkStrategy.noWatermarks(), "planDaySource_kafka");
@@ -295,7 +147,7 @@ public class PlanStatJob {
                             LocalDate endTime = LocalDate.now(), beginTime = statDay.minusDays(60);
                             String sql = "SELECT * FROM plan_stat_of_day_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginTime) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endTime) + "\" AND campaign_id = " + element.getCampaignId() + ";";
                             Instance instance = SQLTask.run(odps, sql);
-                            System.out.println("298===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+                            // log.error("sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
                             instance.waitForSuccess();
                             List<Record> records = SQLTask.getResult(instance);
                             Map<String, PlanStatOfDayDWD> historyData = records.stream()
@@ -318,7 +170,6 @@ public class PlanStatJob {
                         collector.collect(newStatData);
                     }
                 });
-        //.addSink(new TunnelBatchSink<>(AdStatOfDayDWD.class, 30000L, 365L, 6));
         new KeyedBatchStream<>("planDayDWDStream", planDayDWDStream.keyBy(PlanStatOfDayDWD::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(PlanStatOfDayDWD.class))
@@ -343,12 +194,11 @@ public class PlanStatJob {
                         lastReduceState.update(newStatDWD);
                     }
                 });
-        //.addSink(new TunnelBatchSink<>(AdStatOfDayDWD.class, 30000L, 365L, 6));
         new KeyedBatchStream<>("planDayDWDYearStream", planDayDWDYearStream.keyBy(PlanStatOfDayDWD::getStatDay), 4000L, 60 * 1000L)
                 .toBatch()
                 .addSink(new TunnelBatchStreamSink<>(PlanStatOfDayDWD.class))
                 .name("sink_plan_year_dwd");
 
-        env.execute("job_plan");
+        env.execute("plan_day_stream_job");
     }
 }

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/PlanHourStreamJob.java

@@ -0,0 +1,91 @@
+package flink.zanxiangnet.ad.monitoring;
+
+import flink.zanxiangnet.ad.monitoring.pojo.entity.*;
+import flink.zanxiangnet.ad.monitoring.process.PlanHourDTOStreamProcess;
+import flink.zanxiangnet.ad.monitoring.process.PlanHourDWDProcess;
+import flink.zanxiangnet.ad.monitoring.process.PlanMinuteDWDProcess;
+import flink.zanxiangnet.ad.monitoring.sink.TunnelBatchStreamSink;
+import flink.zanxiangnet.ad.monitoring.stream.KeyedBatchStream;
+import flink.zanxiangnet.ad.monitoring.trigger.PlanMinuteODSStreamTrigger;
+import flink.zanxiangnet.ad.monitoring.kafka.KafkaComponent;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.kafka.source.KafkaSource;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.OutputTag;
+
+import java.time.Duration;
+import java.util.*;
+
+public class PlanHourStreamJob {
+
+    /**
+     * 可能有数据的最早日期
+     */
+    private static final String OLDEST_DAY = "2019-01-01";
+
+    public static void main(String[] args) throws Exception {
+        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+        // 加载配置文件到 flink的全局配置中
+        Properties props = new Properties();
+        props.load(PlanHourStreamJob.class.getResourceAsStream("/application.properties"));
+        Configuration configuration = new Configuration();
+        props.stringPropertyNames().forEach(key -> {
+            String value = props.getProperty(key);
+            configuration.setString(key.trim(), StringUtils.isBlank(value) ? "" : value.trim());
+        });
+        env.getConfig().setGlobalJobParameters(configuration);
+
+        KafkaSource<String> adStreamOfMinuteSource = KafkaComponent.buildKafkaSource(props, KafkaComponent.KafkaTopic.adHourTopic, KafkaComponent.KafkaTopic.KafkaGroupId.planHourConsumerGroup);
+        DataStreamSource<String> adStreamOfMinuteIn = env.fromSource(adStreamOfMinuteSource, WatermarkStrategy.noWatermarks(), "planHourSource_kafka");
+
+        // 广告分钟数据（前 5分钟的广告消耗数据）
+        final OutputTag<AdDataOfMinuteODS> adMinuteStreamTag = new OutputTag<AdDataOfMinuteODS>("adMinuteStream") {
+        };
+        // 广告小时数据（往前回滚 10天）
+        final OutputTag<AdDataOfHourODS> adHourStreamTag = new OutputTag<AdDataOfHourODS>("adHourStream") {
+        };
+        // 计划小时数据（从分钟数据流切出来的整点数据）
+        final OutputTag<PlanStatOfHourDWD> planHourFromMinuteStreamTag = new OutputTag<PlanStatOfHourDWD>("planHourFromMinuteStream") {
+        };
+
+        // 对流进行映射，拆分（实时的分钟流和回滚的小时流）
+        SingleOutputStreamOperator<AdDataOfMinuteODS> adODSStream = adStreamOfMinuteIn.filter(StringUtils::isNotBlank)
+                .process(new PlanHourDTOStreamProcess(adMinuteStreamTag, adHourStreamTag));
+
+        // 分钟流-计算
+        SingleOutputStreamOperator<PlanStatOfMinuteDWD> planMinuteDWDStream = adODSStream.getSideOutput(adMinuteStreamTag)
+                // 打水印，允许数据延迟 6分钟，同时指定时间流
+                .assignTimestampsAndWatermarks(WatermarkStrategy.<AdDataOfMinuteODS>forBoundedOutOfOrderness(Duration.ofMinutes(6L))
+                        .withTimestampAssigner((SerializableTimestampAssigner<AdDataOfMinuteODS>) (adODS, l) -> adODS.getStatTime()))
+                .keyBy(AdDataOfMinuteODS::getCampaignId)
+                // 开一个 5分钟的滚动窗口
+                .window(TumblingEventTimeWindows.of(Time.minutes(5L)))
+                .trigger(new PlanMinuteODSStreamTrigger())
+                .process(new PlanMinuteDWDProcess(planHourFromMinuteStreamTag));
+        new KeyedBatchStream<>("planMinuteDWDStream", planMinuteDWDStream.keyBy(PlanStatOfMinuteDWD::getStatDay), 4000L, 60 * 1000L)
+                .toBatch()
+                .addSink(new TunnelBatchStreamSink<>(PlanStatOfMinuteDWD.class))
+                .name("sink_plan_minute_dwd");
+
+        // 小时流
+        SingleOutputStreamOperator<PlanStatOfHourDWD> planHourDWDStream = adODSStream.getSideOutput(adHourStreamTag).keyBy(AdDataOfHourODS::getCampaignId)
+                .countWindow(1)
+                .process(new PlanHourDWDProcess());
+        DataStream<PlanStatOfHourDWD> planHourDWDAllStream = planMinuteDWDStream.getSideOutput(planHourFromMinuteStreamTag).union(planHourDWDStream);
+        new KeyedBatchStream<>("planHourDWDStream", planHourDWDAllStream.keyBy(PlanStatOfHourDWD::getStatDay), 4000L, 60 * 1000L)
+                .toBatch()
+                .addSink(new TunnelBatchStreamSink<>(PlanStatOfHourDWD.class))
+                .name("sink_plan_hour_dwd");
+
+        env.execute("plan_hour_stream_job");
+    }
+}

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/Test.java

@@ -9,6 +9,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.state.MapState;
@@ -17,13 +18,19 @@ import org.apache.flink.api.common.typeinfo.Types;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.streaming.api.windowing.triggers.Trigger;
+import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
 import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
 
 import java.time.Duration;
@@ -38,6 +45,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
+@Slf4j
 public class Test {
     public static void main(String[] args) throws Exception {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
@@ -49,9 +57,90 @@ public class Test {
         SingleOutputStreamOperator<Pojo> pojoStream = source.assignTimestampsAndWatermarks(WatermarkStrategy.<Pojo>forBoundedOutOfOrderness(Duration.ofHours(2))
                 .withTimestampAssigner((SerializableTimestampAssigner<Pojo>) (pojo, l) -> pojo.getCreateTime())
         );
-        /*pojoStream.keyBy(Pojo::getUserId)
+        pojoStream.keyBy(Pojo::getUserId)
                 .window(TumblingEventTimeWindows.of(Time.days(1L), Time.hours(-8)))
-                .aggregate(new AggregateFunction<Pojo, Tuple5<Integer, Long, Long, Integer, List<Long>>, String>() {
+                .trigger(new Trigger<Pojo, TimeWindow>() {
+                    /**
+                     *
+                     * @param pojo
+                     * @param time 触发窗口的时间，比如此例中是 eventTime
+                     * @param timeWindow
+                     * @param triggerContext
+                     * @return
+                     * @throws Exception
+                     */
+                    @Override
+                    public TriggerResult onElement(Pojo pojo, long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+                        if (timeWindow.maxTimestamp() <= triggerContext.getCurrentWatermark()) {
+                            // 到了窗口的最大生命周期
+                            log.error("到了窗口的最大生命周期");
+                            return TriggerResult.FIRE_AND_PURGE;
+                        }
+                        log.error("trigger->onElement: {}, {}, {}, {}, [{} - {}]", JsonUtil.toString(pojo),
+                                DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(timeWindow.getStart())),
+                                DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(timeWindow.getEnd())),
+                                DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(triggerContext.getCurrentWatermark())),
+                                DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(timeWindow.maxTimestamp())),
+                                DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(triggerContext.getCurrentProcessingTime()))
+                        );
+                        return TriggerResult.FIRE;
+                    }
+
+                    /**
+                     * ProcessingTime定时器触发的时候调用
+                     *
+                     * @param time 调用这个方法的时间
+                     * @param timeWindow
+                     * @param triggerContext
+                     * @return
+                     * @throws Exception
+                     */
+                    @Override
+                    public TriggerResult onProcessingTime(long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+                        log.error("trigger->onProcessingTime: {}", DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(time)));
+                        return TriggerResult.CONTINUE;
+                    }
+
+                    /**
+                     * EventTime定时器触发的时候调用
+                     *
+                     * @param time eventTime
+                     * @param timeWindow
+                     * @param triggerContext
+                     * @return
+                     * @throws Exception
+                     */
+                    @Override
+                    public TriggerResult onEventTime(long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+                        log.error("trigger->onEventTime: {}", DateUtil.formatLocalDateTime(DateUtil.milliToLocalDateTime(time)));
+                        return TriggerResult.PURGE;
+                    }
+
+                    /**
+                     * 窗口处理结束后，清除窗口的时候调用
+                     *
+                     * @param timeWindow
+                     * @param triggerContext
+                     * @throws Exception
+                     */
+                    @Override
+                    public void clear(TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+                        log.error("trigger->clear");
+                    }
+                })
+                .process(new ProcessWindowFunction<Pojo, Tuple5<Integer, Long, Long, Integer, List<Long>>, Integer, TimeWindow>() {
+                    @Override
+                    public void process(Integer integer,
+                                        ProcessWindowFunction<Pojo, Tuple5<Integer, Long, Long, Integer, List<Long>>, Integer, TimeWindow>.Context context, Iterable<Pojo> iterable,
+                                        Collector<Tuple5<Integer, Long, Long, Integer, List<Long>>> collector) throws Exception {
+                        log.error("----------------- begin -------------------------");
+                        for (Pojo pojo : iterable) {
+                            log.error("收到元素：" + pojo);
+                        }
+                        log.error("----------------- end --------------------------");
+                    }
+                }).print();
+                /*.aggregate(new AggregateFunction<Pojo, Tuple5<Integer, Long, Long, Integer, List<Long>>, String>() {
 
                     @Override
                     public Tuple5<Integer, Long, Long, Integer, List<Long>> createAccumulator() {
@@ -76,11 +165,12 @@ public class Test {
                         return new Tuple5<>(tuple5.f0, Math.min(acc.f1, tuple5.f1), Math.max(acc.f2, tuple5.f2), tuple5.f3 + acc.f3, temp);
                     }
                 }).print();*/
-        SingleOutputStreamOperator<String> ff = pojoStream.keyBy(Pojo::getUserId).countWindow(1)
+        /*SingleOutputStreamOperator<String> ff = pojoStream.keyBy(Pojo::getUserId).countWindow(1)
                 .process(new ProcessWindowFunction<Pojo, String, Integer, GlobalWindow>() {
                     private final Map<Integer, Tuple3<LocalDateTime, LocalDateTime, List<Pojo>>> oldData = new ConcurrentHashMap<>();
 
                     private MapState<Integer, Tuple3<LocalDateTime, LocalDateTime, List<Pojo>>> mapState;
+
                     @Override
                     public void open(Configuration conf) {
                         mapState = getRuntimeContext().getMapState(new MapStateDescriptor<>("testState", Types.INT, Types.TUPLE(Types.LOCAL_DATE_TIME, Types.LOCAL_DATE_TIME, Types.LIST(Types.POJO(Pojo.class)))));
@@ -93,8 +183,8 @@ public class Test {
                         LocalDateTime createTime = DateUtil.milliToLocalDateTime(pojo.getCreateTime());
                         // Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> temp = oldData.get(pojo.getUserId());
                         Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> temp = mapState.get(pojo.getUserId());
-                        System.out.println(mapState.get(1));
-                        System.out.println(mapState.get(2));
+                        log.error(mapState.get(1));
+                        log.error(mapState.get(2));
                         if (temp == null || createTime.getDayOfYear() != temp.f1.getDayOfYear()) {
                             temp = new Tuple3<>(createTime, createTime, new ArrayList<>(200));
                         }
@@ -104,25 +194,7 @@ public class Test {
                         out.collect(JsonUtil.toString(temp.f2.stream().map(Pojo::getIndex).collect(Collectors.toList())));
                     }
                 });
-        new BatchStream<>("ff", ff, 10L, 60 * 1000L).toBatch().print();
-        // new BatchSink<>(ff, 60 * 1000L, 10L).toBatch().print();
-/*        .apply(new WindowFunction<Pojo, String, Integer, GlobalWindow>() {
-            private Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> oldData = null;
-            @Override
-            public void apply(Integer integer, GlobalWindow window, Iterable<Pojo> input, Collector<String> out) throws Exception {
-                Pojo pojo = null;
-                for(Pojo obj : input) {
-                    pojo = obj;
-                }
-                LocalDateTime createTime = DateUtil.milliToLocalDateTime(pojo.getCreateTime());
-                if (oldData == null || createTime.getDayOfYear() != oldData.f1.getDayOfYear()) {
-                    oldData = new Tuple3<>(createTime, createTime, new ArrayList<>(200));
-                }
-                oldData.f2.add(pojo);
-                oldData.setField(createTime, 1);
-                out.collect(JsonUtil.toString(oldData.f2.stream().map(Pojo::getIndex).collect(Collectors.toList())));
-            }
-        })*/
+        new BatchStream<>("ff", ff, 10L, 60 * 1000L).toBatch().print();*/
 
         env.execute();
     }
@@ -148,7 +220,7 @@ public class Test {
             if (index1 == null) {
                 synchronized (DUMMY_LOCK) {
                     if (index1 == null) {
-                        index1 = new AtomicLong(12);
+                        index1 = new AtomicLong(0);
                         index2 = new AtomicLong(0);
                         threadPool = new ThreadPoolExecutor(
                                 60,
@@ -167,13 +239,13 @@ public class Test {
         public void run(SourceContext<Pojo> sourceContext) {
             while (isRun) {
                 try {
-                    long user1 = index1.incrementAndGet();
+                    long user1Index = index1.incrementAndGet();
                     Pojo pojo = Pojo.builder()
                             .userId(1)
-                            .index(user1)
-                            .createTime(BEGIN + ((user1 - 1) * 60 * 60 * 1000))
+                            .index(user1Index)
+                            .createTime(BEGIN + ((user1Index - 1) * 60 * 60 * 1000))
                             .build();
-                    if (user1 % 24 == 0) {
+                    if (user1Index % 24 == 0) {
                         // 模拟数据延迟，每天 24点的数据延迟 25秒
                         threadPool.execute(() -> {
                             try {
@@ -181,20 +253,20 @@ public class Test {
                             } catch (InterruptedException e) {
                                 e.printStackTrace();
                             }
-                            System.out.println("延迟1发送： " + pojo);
+                            log.error("延迟1发送：{}", pojo);
                             sourceContext.collect(pojo);
                         });
                     } else {
-                        System.out.println("1发送： " + pojo);
+                        // log.error("1发送：{}", pojo);
                         sourceContext.collect(pojo);
                     }
-                    long user2 = index2.incrementAndGet();
+                    /*long user2Index = index2.incrementAndGet();
                     Pojo pojo2 = Pojo.builder()
                             .userId(2)
-                            .index(user2)
-                            .createTime(BEGIN + ((user2 - 1) * 60 * 60 * 1000))
+                            .index(user2Index)
+                            .createTime(BEGIN + ((user2Index - 1) * 60 * 60 * 1000))
                             .build();
-                    if (user2 % 24 == 0) {
+                    if (user2Index % 24 == 0) {
                         // 模拟数据延迟，每天 24点的数据延迟 25秒
                         threadPool.execute(() -> {
                             try {
@@ -202,13 +274,13 @@ public class Test {
                             } catch (InterruptedException e) {
                                 e.printStackTrace();
                             }
-                            System.out.println("延迟2发送： " + pojo2);
+                            log.error("延迟2发送：{}", pojo2);
                             sourceContext.collect(pojo2);
                         });
                     } else {
-                        System.out.println("2发送： " + pojo2);
+                        // log.error("2发送：{}", pojo2);
                         sourceContext.collect(pojo2);
-                    }
+                    }*/
                     Thread.sleep(1000);
                 } catch (Exception e) {
                     e.printStackTrace();

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/clickhouse/sink/BatchSinkHour.java

@@ -2,6 +2,7 @@ package flink.zanxiangnet.ad.monitoring.clickhouse.sink;
 
 import flink.zanxiangnet.ad.monitoring.AdStatJob;
 import flink.zanxiangnet.ad.monitoring.pojo.entity.CostHourDM;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.slf4j.Logger;
@@ -13,8 +14,8 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+@Slf4j
 public class BatchSinkHour extends RichSinkFunction<CostHourDM> {
-    private static final Logger log = LoggerFactory.getLogger(BatchSinkHour.class);
 
     // 对象锁，防止MaxCompute的 Tunnel对象多次初始化
     private static final Object DUMMY_LOCK = new Object();
@@ -133,7 +134,7 @@ public class BatchSinkHour extends RichSinkFunction<CostHourDM> {
                 "?,?,?,?,?,?,?,?,?,?,\n" +
                 "?,?,?,?,?,?,?,?,?,?,\n" +
                 "?,?,?,?,?,?,?)";
-        System.out.println(costhour);
+        log.error(costhour.toString());
         PreparedStatement preparedStatement = connection.prepareStatement(sql);
         preparedStatement.setString(1, costhour.dt);
         preparedStatement.setString(2, costhour.createTime);
@@ -227,7 +228,7 @@ public class BatchSinkHour extends RichSinkFunction<CostHourDM> {
         int ints[] = preparedStatement.executeBatch();
         connection.commit();
         long endTime = System.currentTimeMillis();
-        System.out.println("批量插入耗时: " + (endTime - startTime) + "插入数据数量=" + ints.length);
+        log.error("批量插入耗时: " + (endTime - startTime) + "插入数据数量=" + ints.length);
         //clickhouse 处理重复数据
         //TODO:数据去重有问题，去除掉非最新的数据
 //        Statement statement_duplicate = connection.createStatement();
@@ -235,7 +236,7 @@ public class BatchSinkHour extends RichSinkFunction<CostHourDM> {
 //        statement_duplicate.executeQuery(sql_duplicate);
 //        connection.commit();
         long endTime_dp = System.currentTimeMillis();
-        System.out.println("数据清理耗时: " + (endTime_dp - endTime));
+        log.error("数据清理耗时: " + (endTime_dp - endTime));
 
     }
 

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/clickhouse/sink/BatchSinkMinute.java

@@ -2,6 +2,7 @@ package flink.zanxiangnet.ad.monitoring.clickhouse.sink;
 
 import flink.zanxiangnet.ad.monitoring.AdStatJob;
 import flink.zanxiangnet.ad.monitoring.pojo.entity.CostMinuterDM;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 import org.slf4j.Logger;
@@ -13,8 +14,8 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+@Slf4j
 public class BatchSinkMinute extends RichSinkFunction<CostMinuterDM> {
-    private static final Logger log = LoggerFactory.getLogger(BatchSinkMinute.class);
 
     // 对象锁，防止MaxCompute的 Tunnel对象多次初始化
     private static final Object DUMMY_LOCK = new Object();
@@ -82,7 +83,7 @@ public class BatchSinkMinute extends RichSinkFunction<CostMinuterDM> {
                 "?,?,?,?,?,?,?,?,?,?,\n" +
                 "?,?,?,?,?,?,?,?,?,?,\n" +
                 "?,?,?,?,?,?,?,?,?,?,\n)";
-//        System.out.println(costMinuterDM);
+//        log.error(costMinuterDM);
         PreparedStatement preparedStatement = connection.prepareStatement(sql);
         preparedStatement.setString(1, costMinuterDM.dt);
         preparedStatement.setString(2, costMinuterDM.minute);
@@ -130,14 +131,14 @@ public class BatchSinkMinute extends RichSinkFunction<CostMinuterDM> {
         int ints[] = preparedStatement.executeBatch();
         connection.commit();
         long endTime = System.currentTimeMillis();
-        System.out.println("批量插入耗时: " + (endTime - startTime) + "插入数据数量=" + ints.length);
+        log.error("批量插入耗时: " + (endTime - startTime) + "插入数据数量=" + ints.length);
         //TODO:数据去重有问题，去除掉非最新的数据
 //        Statement statement_duplicate = connection.createStatement();
 //        String sql_duplicate = "optimize table data_monitoring.cost_minute final;";
 //        statement_duplicate.executeQuery(sql_duplicate);
 //        connection.commit();
         long endTime_dp = System.currentTimeMillis();
-        System.out.println("数据清理耗时: " + (endTime_dp - endTime));
+        log.error("数据清理耗时: " + (endTime_dp - endTime));
 
     }
 

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/pojo/entity/AdStatOfHourDWD.java

@@ -2,6 +2,7 @@ package flink.zanxiangnet.ad.monitoring.pojo.entity;
 
 import com.aliyun.odps.data.Record;
 import com.google.gson.annotations.SerializedName;
+import flink.zanxiangnet.ad.monitoring.maxcompute.bean.BeanUtil;
 import flink.zanxiangnet.ad.monitoring.maxcompute.bean.annotation.MaxComputeColumn;
 import flink.zanxiangnet.ad.monitoring.maxcompute.bean.annotation.MaxComputeTable;
 import flink.zanxiangnet.ad.monitoring.util.ObjectUtil;
@@ -1200,6 +1201,12 @@ public class AdStatOfHourDWD {
     @SerializedName("no_interest_count_hour")
     private Long noInterestCountHour;
 
+    public static AdStatOfHourDWD byMinuteDWD(AdStatOfMinuteDWD minuteDWD) {
+        AdStatOfHourDWD result = new AdStatOfHourDWD();
+        BeanUtils.copyProperties(minuteDWD, result);
+        return result;
+    }
+
     public static AdStatOfHourDWD byODS(AdDataOfHourODS adODS) {
         AdStatOfHourDWD adStatOfHour = new AdStatOfHourDWD();
         adStatOfHour.setStatDay(adODS.getStatDay());

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/pojo/entity/AdStatOfMinuteDWD.java

@@ -1828,42 +1828,42 @@ public class AdStatOfMinuteDWD {
             result.setNoInterestCountTotal(beforeYesterdayDayDWD.getNoInterestCountTotal());
         }
         if (yesterdayMinuteDWD != null) {
-            result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + yesterdayMinuteDWD.getCostDeviationRateTotal());
-            result.setCostTotal(result.getCostTotal() + yesterdayMinuteDWD.getCostTotal());
-            result.setCompensationAmountTotal(result.getCompensationAmountTotal() + yesterdayMinuteDWD.getCompensationAmountTotal());
-            result.setViewCountTotal(result.getViewCountTotal() + yesterdayMinuteDWD.getCompensationAmountTotal());
-            result.setValidClickCountTotal(result.getValidClickCountTotal() + yesterdayMinuteDWD.getValidClickCountTotal());
-            result.setValuableClickCountTotal(result.getValuableClickCountTotal() + yesterdayMinuteDWD.getValuableClickCountTotal());
-            result.setConversionsCountTotal(result.getConversionsCountTotal() + yesterdayMinuteDWD.getConversionsCountTotal());
-            result.setDeepConversionsCountTotal(result.getDeepConversionsCountTotal() + yesterdayMinuteDWD.getDeepConversionsCountTotal());
-            result.setOrderCountTotal(result.getOrderCountTotal() + yesterdayMinuteDWD.getOrderCountTotal());
-            result.setFirstDayOrderCountTotal(result.getFirstDayOrderCountTotal() + yesterdayMinuteDWD.getFirstDayOrderCountTotal());
-            result.setOrderAmountTotal(result.getOrderAmountTotal() + yesterdayMinuteDWD.getOrderAmountTotal());
-            result.setFirstDayOrderAmountTotal(result.getFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getFirstDayOrderAmountTotal());
-            result.setSignInCountTotal(result.getSignInCountTotal() + yesterdayMinuteDWD.getSignInCountTotal());
-            result.setScanFollowCountTotal(result.getScanFollowCountTotal() + yesterdayMinuteDWD.getScanFollowCountTotal());
-            result.setWechatAppRegisterUvTotal(result.getWechatAppRegisterUvTotal() + yesterdayMinuteDWD.getWechatAppRegisterUvTotal());
-            result.setWechatMinigameRetentionCountTotal(result.getWechatMinigameRetentionCountTotal() + yesterdayMinuteDWD.getWechatMinigameRetentionCountTotal());
-            result.setWechatMinigameCheckoutCountTotal(result.getWechatMinigameCheckoutCountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutCountTotal());
-            result.setWechatMinigameCheckoutAmountTotal(result.getWechatMinigameCheckoutAmountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutAmountTotal());
-            result.setOfficialAccountFollowCountTotal(result.getOfficialAccountFollowCountTotal() + yesterdayMinuteDWD.getOfficialAccountFollowCountTotal());
-            result.setOfficialAccountRegisterUserCountTotal(result.getOfficialAccountRegisterUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterUserCountTotal());
-            result.setOfficialAccountRegisterAmountTotal(result.getOfficialAccountRegisterAmountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterAmountTotal());
-            result.setOfficialAccountApplyCountTotal(result.getOfficialAccountApplyCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyCountTotal());
-            result.setOfficialAccountApplyUserCountTotal(result.getOfficialAccountApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyUserCountTotal());
-            result.setOfficialAccountApplyAmountTotal(result.getOfficialAccountApplyAmountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyAmountTotal());
-            result.setOfficialAccountOrderCountTotal(result.getOfficialAccountOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderCountTotal());
-            result.setOfficialAccountFirstDayOrderCountTotal(result.getOfficialAccountFirstDayOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderCountTotal());
-            result.setOfficialAccountOrderUserCountTotal(result.getOfficialAccountOrderUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderUserCountTotal());
-            result.setOfficialAccountOrderAmountTotal(result.getOfficialAccountOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderAmountTotal());
-            result.setOfficialAccountFirstDayOrderAmountTotal(result.getOfficialAccountFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderAmountTotal());
-            result.setOfficialAccountConsultCountTotal(result.getOfficialAccountConsultCountTotal() + yesterdayMinuteDWD.getOfficialAccountConsultCountTotal());
-            result.setOfficialAccountReaderCountTotal(result.getOfficialAccountReaderCountTotal() + yesterdayMinuteDWD.getOfficialAccountReaderCountTotal());
-            result.setOfficialAccountCreditApplyUserCountTotal(result.getOfficialAccountCreditApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditApplyUserCountTotal());
-            result.setOfficialAccountCreditUserCountTotal(result.getOfficialAccountCreditUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditUserCountTotal());
-            result.setForwardCountTotal(result.getForwardCountTotal() + yesterdayMinuteDWD.getForwardCountTotal());
-            result.setForwardUserCountTotal(result.getForwardUserCountTotal() + yesterdayMinuteDWD.getForwardUserCountTotal());
-            result.setNoInterestCountTotal(result.getNoInterestCountTotal() + yesterdayMinuteDWD.getNoInterestCountTotal());
+            result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + yesterdayMinuteDWD.getCostDeviationRateDay());
+            result.setCostTotal(result.getCostTotal() + yesterdayMinuteDWD.getCostDay());
+            result.setCompensationAmountTotal(result.getCompensationAmountTotal() + yesterdayMinuteDWD.getCompensationAmountDay());
+            result.setViewCountTotal(result.getViewCountTotal() + yesterdayMinuteDWD.getCompensationAmountDay());
+            result.setValidClickCountTotal(result.getValidClickCountTotal() + yesterdayMinuteDWD.getValidClickCountDay());
+            result.setValuableClickCountTotal(result.getValuableClickCountTotal() + yesterdayMinuteDWD.getValuableClickCountDay());
+            result.setConversionsCountTotal(result.getConversionsCountTotal() + yesterdayMinuteDWD.getConversionsCountDay());
+            result.setDeepConversionsCountTotal(result.getDeepConversionsCountTotal() + yesterdayMinuteDWD.getDeepConversionsCountDay());
+            result.setOrderCountTotal(result.getOrderCountTotal() + yesterdayMinuteDWD.getOrderCountDay());
+            result.setFirstDayOrderCountTotal(result.getFirstDayOrderCountTotal() + yesterdayMinuteDWD.getFirstDayOrderCountDay());
+            result.setOrderAmountTotal(result.getOrderAmountTotal() + yesterdayMinuteDWD.getOrderAmountDay());
+            result.setFirstDayOrderAmountTotal(result.getFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getFirstDayOrderAmountDay());
+            result.setSignInCountTotal(result.getSignInCountTotal() + yesterdayMinuteDWD.getSignInCountDay());
+            result.setScanFollowCountTotal(result.getScanFollowCountTotal() + yesterdayMinuteDWD.getScanFollowCountDay());
+            result.setWechatAppRegisterUvTotal(result.getWechatAppRegisterUvTotal() + yesterdayMinuteDWD.getWechatAppRegisterUvDay());
+            result.setWechatMinigameRetentionCountTotal(result.getWechatMinigameRetentionCountTotal() + yesterdayMinuteDWD.getWechatMinigameRetentionCountDay());
+            result.setWechatMinigameCheckoutCountTotal(result.getWechatMinigameCheckoutCountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutCountDay());
+            result.setWechatMinigameCheckoutAmountTotal(result.getWechatMinigameCheckoutAmountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutAmountDay());
+            result.setOfficialAccountFollowCountTotal(result.getOfficialAccountFollowCountTotal() + yesterdayMinuteDWD.getOfficialAccountFollowCountDay());
+            result.setOfficialAccountRegisterUserCountTotal(result.getOfficialAccountRegisterUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterUserCountDay());
+            result.setOfficialAccountRegisterAmountTotal(result.getOfficialAccountRegisterAmountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterAmountDay());
+            result.setOfficialAccountApplyCountTotal(result.getOfficialAccountApplyCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyCountDay());
+            result.setOfficialAccountApplyUserCountTotal(result.getOfficialAccountApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyUserCountDay());
+            result.setOfficialAccountApplyAmountTotal(result.getOfficialAccountApplyAmountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyAmountDay());
+            result.setOfficialAccountOrderCountTotal(result.getOfficialAccountOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderCountDay());
+            result.setOfficialAccountFirstDayOrderCountTotal(result.getOfficialAccountFirstDayOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderCountDay());
+            result.setOfficialAccountOrderUserCountTotal(result.getOfficialAccountOrderUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderUserCountDay());
+            result.setOfficialAccountOrderAmountTotal(result.getOfficialAccountOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderAmountDay());
+            result.setOfficialAccountFirstDayOrderAmountTotal(result.getOfficialAccountFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderAmountDay());
+            result.setOfficialAccountConsultCountTotal(result.getOfficialAccountConsultCountTotal() + yesterdayMinuteDWD.getOfficialAccountConsultCountDay());
+            result.setOfficialAccountReaderCountTotal(result.getOfficialAccountReaderCountTotal() + yesterdayMinuteDWD.getOfficialAccountReaderCountDay());
+            result.setOfficialAccountCreditApplyUserCountTotal(result.getOfficialAccountCreditApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditApplyUserCountDay());
+            result.setOfficialAccountCreditUserCountTotal(result.getOfficialAccountCreditUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditUserCountDay());
+            result.setForwardCountTotal(result.getForwardCountTotal() + yesterdayMinuteDWD.getForwardCountDay());
+            result.setForwardUserCountTotal(result.getForwardUserCountTotal() + yesterdayMinuteDWD.getForwardUserCountDay());
+            result.setNoInterestCountTotal(result.getNoInterestCountTotal() + yesterdayMinuteDWD.getNoInterestCountDay());
         }
         result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + result.getCostDeviationRateDay());
         result.setCostTotal(result.getCostTotal() + result.getCostDay());
@@ -1955,136 +1955,134 @@ public class AdStatOfMinuteDWD {
         result.setOfficialAccountOrderRoiAll(result.getCostTotal() == 0 ? 0 : result.getOfficialAccountOrderAmountTotal() / result.getCostTotal());
 
         // 上一次聚合的 5分钟数据
-        if (lastMinuteDWD != null) {
-            if (lastMinuteDWD.getHour().equals(result.getHour())) {
-                result.setCostDeviationRateMinute(result.getCostDeviationRateHour() - lastMinuteDWD.getCostDeviationRateHour());
-                result.setCostMinute(result.getCostHour() - lastMinuteDWD.getCostHour());
-                result.setCompensationAmountMinute(result.getCompensationAmountHour() - lastMinuteDWD.getCompensationAmountHour());
-                result.setViewCountMinute(result.getViewCountHour() - lastMinuteDWD.getViewCountHour());
-                result.setThousandDisplayPriceMinute(result.getThousandDisplayPriceHour() - lastMinuteDWD.getThousandDisplayPriceHour());
-                result.setAvgViewPerUserMinute(result.getAvgViewPerUserHour() - lastMinuteDWD.getAvgViewPerUserHour());
-                result.setValidClickCountMinute(result.getValidClickCountHour() - lastMinuteDWD.getValidClickCountHour());
-                result.setCtrMinute(result.getCtrHour() - lastMinuteDWD.getCtrHour());
-                result.setCpcMinute(result.getCpcHour() - lastMinuteDWD.getCpcHour());
-                result.setValuableClickCountMinute(result.getValuableClickCountHour() - lastMinuteDWD.getValuableClickCountHour());
-                result.setValuableClickRateMinute(result.getValuableClickRateHour() - lastMinuteDWD.getValuableClickRateHour());
-                result.setValuableClickCostMinute(result.getValuableClickCostHour() - lastMinuteDWD.getValuableClickCostHour());
-                result.setConversionsCountMinute(result.getConversionsCountHour() - lastMinuteDWD.getConversionsCountHour());
-                result.setConversionsCostMinute(result.getConversionsCostHour() - lastMinuteDWD.getConversionsCostHour());
-                result.setConversionsRateMinute(result.getConversionsRateHour() - lastMinuteDWD.getConversionsRateHour());
-                result.setDeepConversionsCountMinute(result.getDeepConversionsCountHour() - lastMinuteDWD.getDeepConversionsCountHour());
-                result.setDeepConversionsCostMinute(result.getDeepConversionsCostHour() - lastMinuteDWD.getDeepConversionsCostHour());
-                result.setDeepConversionsRateMinute(result.getDeepConversionsRateHour() - lastMinuteDWD.getDeepConversionsRateHour());
-                result.setOrderCountMinute(result.getOrderCountHour() - lastMinuteDWD.getOrderCountHour());
-                result.setFirstDayOrderCountMinute(result.getFirstDayOrderCountHour() - lastMinuteDWD.getFirstDayOrderCountHour());
-                result.setWebOrderCostMinute(result.getWebOrderCostHour() - lastMinuteDWD.getWebOrderCostHour());
-                result.setOrderRateMinute(result.getOrderRateHour() - lastMinuteDWD.getOrderRateHour());
-                result.setOrderAmountMinute(result.getOrderAmountHour() - lastMinuteDWD.getOrderAmountHour());
-                result.setFirstDayOrderAmountMinute(result.getFirstDayOrderAmountHour() - lastMinuteDWD.getFirstDayOrderAmountHour());
-                result.setOrderUnitPriceMinute(result.getOrderUnitPriceHour() - lastMinuteDWD.getOrderUnitPriceHour());
-                result.setOrderRoiMinute(result.getOrderRoiHour() - lastMinuteDWD.getOrderRoiHour());
-                result.setSignInCountMinute(result.getSignInCountHour() - lastMinuteDWD.getSignInCountHour());
-                result.setScanFollowCountMinute(result.getScanFollowCountHour() - lastMinuteDWD.getScanFollowCountHour());
-                result.setWechatAppRegisterUvMinute(result.getWechatAppRegisterUvHour() - lastMinuteDWD.getWechatAppRegisterUvHour());
-                result.setWechatMinigameRegisterCostMinute(result.getWechatMinigameRegisterCostHour() - lastMinuteDWD.getWechatMinigameRegisterCostHour());
-                result.setWechatMinigameRegisterRateMinute(result.getWechatMinigameRegisterRateHour() - lastMinuteDWD.getWechatMinigameRegisterRateHour());
-                result.setWechatMinigameArpuMinute(result.getWechatMinigameArpuHour() - lastMinuteDWD.getWechatMinigameArpuHour());
-                result.setWechatMinigameRetentionCountMinute(result.getWechatMinigameRetentionCountHour() - lastMinuteDWD.getWechatMinigameRetentionCountHour());
-                result.setWechatMinigameCheckoutCountMinute(result.getWechatMinigameCheckoutCountHour() - lastMinuteDWD.getWechatMinigameCheckoutCountHour());
-                result.setWechatMinigameCheckoutAmountMinute(result.getWechatMinigameCheckoutAmountHour() - lastMinuteDWD.getWechatMinigameCheckoutAmountHour());
-                result.setOfficialAccountFollowCountMinute(result.getOfficialAccountFollowCountHour() - lastMinuteDWD.getOfficialAccountFollowCountHour());
-                result.setOfficialAccountFollowRateMinute(result.getOfficialAccountFollowRateHour() - lastMinuteDWD.getOfficialAccountFollowRateHour());
-                result.setOfficialAccountRegisterUserCountMinute(result.getOfficialAccountRegisterUserCountHour() - lastMinuteDWD.getOfficialAccountRegisterUserCountHour());
-                result.setOfficialAccountRegisterRateMinute(result.getOfficialAccountRegisterRateHour() - lastMinuteDWD.getOfficialAccountRegisterRateHour());
-                result.setOfficialAccountRegisterCostMinute(result.getOfficialAccountRegisterCostHour() - lastMinuteDWD.getOfficialAccountRegisterCostHour());
-                result.setOfficialAccountRegisterAmountMinute(result.getOfficialAccountRegisterAmountHour() - lastMinuteDWD.getOfficialAccountRegisterAmountHour());
-                result.setOfficialAccountRegisterRoiMinute(result.getOfficialAccountRegisterRoiHour() - lastMinuteDWD.getOfficialAccountRegisterRoiHour());
-                result.setOfficialAccountApplyCountMinute(result.getOfficialAccountApplyCountHour() - lastMinuteDWD.getOfficialAccountApplyCountHour());
-                result.setOfficialAccountApplyUserCountMinute(result.getOfficialAccountApplyUserCountHour() - lastMinuteDWD.getOfficialAccountApplyUserCountHour());
-                result.setOfficialAccountApplyRateMinute(result.getOfficialAccountApplyRateHour() - lastMinuteDWD.getOfficialAccountApplyRateHour());
-                result.setOfficialAccountApplyCostMinute(result.getOfficialAccountApplyCostHour() - lastMinuteDWD.getOfficialAccountApplyCostHour());
-                result.setOfficialAccountApplyAmountMinute(result.getOfficialAccountApplyAmountHour() - lastMinuteDWD.getOfficialAccountApplyAmountHour());
-                result.setOfficialAccountApplyRoiMinute(result.getOfficialAccountApplyRoiHour() - lastMinuteDWD.getOfficialAccountApplyRoiHour());
-                result.setOfficialAccountOrderCountMinute(result.getOfficialAccountOrderCountHour() - lastMinuteDWD.getOfficialAccountOrderCountHour());
-                result.setOfficialAccountFirstDayOrderCountMinute(result.getOfficialAccountFirstDayOrderCountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderCountHour());
-                result.setOfficialAccountOrderUserCountMinute(result.getOfficialAccountOrderUserCountHour() - lastMinuteDWD.getOfficialAccountOrderUserCountHour());
-                result.setOfficialAccountOrderRateMinute(result.getOfficialAccountOrderRateHour() - lastMinuteDWD.getOfficialAccountOrderRateHour());
-                result.setOfficialAccountOrderCostMinute(result.getOfficialAccountOrderCostHour() - lastMinuteDWD.getOfficialAccountOrderCostHour());
-                result.setOfficialAccountOrderAmountMinute(result.getOfficialAccountOrderAmountHour() - lastMinuteDWD.getOfficialAccountOrderAmountHour());
-                result.setOfficialAccountFirstDayOrderAmountMinute(result.getOfficialAccountFirstDayOrderAmountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderAmountHour());
-                result.setOfficialAccountOrderRoiMinute(result.getOfficialAccountOrderRoiHour() - lastMinuteDWD.getOfficialAccountOrderRoiHour());
-                result.setOfficialAccountConsultCountMinute(result.getOfficialAccountConsultCountHour() - lastMinuteDWD.getOfficialAccountConsultCountHour());
-                result.setOfficialAccountReaderCountMinute(result.getOfficialAccountReaderCountHour() - lastMinuteDWD.getOfficialAccountReaderCountHour());
-                result.setOfficialAccountCreditApplyUserCountMinute(result.getOfficialAccountCreditApplyUserCountHour() - lastMinuteDWD.getOfficialAccountCreditApplyUserCountHour());
-                result.setOfficialAccountCreditUserCountMinute(result.getOfficialAccountCreditUserCountHour() - lastMinuteDWD.getOfficialAccountCreditUserCountHour());
-                result.setForwardCountMinute(result.getForwardCountHour() - lastMinuteDWD.getForwardCountHour());
-                result.setForwardUserCountMinute(result.getForwardUserCountHour() - lastMinuteDWD.getForwardUserCountHour());
-                result.setNoInterestCountMinute(result.getNoInterestCountHour() - lastMinuteDWD.getNoInterestCountHour());
-            } else {
-                result.setCostDeviationRateMinute(statODS.getCostDeviationRate());
-                result.setCostMinute(statODS.getCost());
-                result.setCompensationAmountMinute(statODS.getCompensationAmount());
-                result.setViewCountMinute(statODS.getViewCount());
-                result.setThousandDisplayPriceMinute(statODS.getThousandDisplayPrice());
-                result.setAvgViewPerUserMinute(statODS.getAvgViewPerUser());
-                result.setValidClickCountMinute(statODS.getValidClickCount());
-                result.setCtrMinute(statODS.getCtr());
-                result.setCpcMinute(statODS.getCpc());
-                result.setValuableClickCountMinute(statODS.getValuableClickCount());
-                result.setValuableClickRateMinute(statODS.getValuableClickRate());
-                result.setValuableClickCostMinute(statODS.getValuableClickCost());
-                result.setConversionsCountMinute(statODS.getConversionsCount());
-                result.setConversionsCostMinute(statODS.getConversionsCost());
-                result.setConversionsRateMinute(statODS.getConversionsRate());
-                result.setDeepConversionsCountMinute(statODS.getDeepConversionsCount());
-                result.setDeepConversionsCostMinute(statODS.getDeepConversionsCost());
-                result.setDeepConversionsRateMinute(statODS.getDeepConversionsRate());
-                result.setOrderCountMinute(statODS.getOrderCount());
-                result.setFirstDayOrderCountMinute(statODS.getFirstDayOrderCount());
-                result.setWebOrderCostMinute(statODS.getWebOrderCost());
-                result.setOrderRateMinute(statODS.getOrderRate());
-                result.setOrderAmountMinute(statODS.getOrderAmount());
-                result.setFirstDayOrderAmountMinute(statODS.getFirstDayOrderAmount());
-                result.setOrderUnitPriceMinute(statODS.getOrderUnitPrice());
-                result.setOrderRoiMinute(statODS.getOrderRoi());
-                result.setSignInCountMinute(statODS.getSignInCount());
-                result.setScanFollowCountMinute(statODS.getScanFollowCount());
-                result.setWechatAppRegisterUvMinute(statODS.getWechatAppRegisterUv());
-                result.setWechatMinigameRegisterCostMinute(statODS.getWechatMinigameRegisterCost());
-                result.setWechatMinigameRegisterRateMinute(statODS.getWechatMinigameRegisterRate());
-                result.setWechatMinigameArpuMinute(statODS.getWechatMinigameArpu());
-                result.setWechatMinigameRetentionCountMinute(statODS.getWechatMinigameRetentionCount());
-                result.setWechatMinigameCheckoutCountMinute(statODS.getWechatMinigameCheckoutCount());
-                result.setWechatMinigameCheckoutAmountMinute(statODS.getWechatMinigameCheckoutAmount());
-                result.setOfficialAccountFollowCountMinute(statODS.getOfficialAccountFollowCount());
-                result.setOfficialAccountFollowRateMinute(statODS.getOfficialAccountFollowRate());
-                result.setOfficialAccountRegisterUserCountMinute(statODS.getOfficialAccountRegisterUserCount());
-                result.setOfficialAccountRegisterRateMinute(statODS.getOfficialAccountRegisterRate());
-                result.setOfficialAccountRegisterCostMinute(statODS.getOfficialAccountRegisterCost());
-                result.setOfficialAccountRegisterAmountMinute(statODS.getOfficialAccountRegisterAmount());
-                result.setOfficialAccountRegisterRoiMinute(statODS.getOfficialAccountRegisterRoi());
-                result.setOfficialAccountApplyCountMinute(statODS.getOfficialAccountApplyCount());
-                result.setOfficialAccountApplyUserCountMinute(statODS.getOfficialAccountApplyUserCount());
-                result.setOfficialAccountApplyRateMinute(statODS.getOfficialAccountApplyRate());
-                result.setOfficialAccountApplyCostMinute(statODS.getOfficialAccountApplyCost());
-                result.setOfficialAccountApplyAmountMinute(statODS.getOfficialAccountApplyAmount());
-                result.setOfficialAccountApplyRoiMinute(statODS.getOfficialAccountApplyRoi());
-                result.setOfficialAccountOrderCountMinute(statODS.getOfficialAccountOrderCount());
-                result.setOfficialAccountFirstDayOrderCountMinute(statODS.getOfficialAccountFirstDayOrderCount());
-                result.setOfficialAccountOrderUserCountMinute(statODS.getOfficialAccountOrderUserCount());
-                result.setOfficialAccountOrderRateMinute(statODS.getOfficialAccountOrderRate());
-                result.setOfficialAccountOrderCostMinute(statODS.getOfficialAccountOrderCost());
-                result.setOfficialAccountOrderAmountMinute(statODS.getOfficialAccountOrderAmount());
-                result.setOfficialAccountFirstDayOrderAmountMinute(statODS.getOfficialAccountFirstDayOrderAmount());
-                result.setOfficialAccountOrderRoiMinute(statODS.getOfficialAccountOrderRoi());
-                result.setOfficialAccountConsultCountMinute(statODS.getOfficialAccountConsultCount());
-                result.setOfficialAccountReaderCountMinute(statODS.getOfficialAccountReaderCount());
-                result.setOfficialAccountCreditApplyUserCountMinute(statODS.getOfficialAccountCreditApplyUserCount());
-                result.setOfficialAccountCreditUserCountMinute(statODS.getOfficialAccountCreditUserCount());
-                result.setForwardCountMinute(statODS.getForwardCount());
-                result.setForwardUserCountMinute(statODS.getForwardUserCount());
-                result.setNoInterestCountMinute(statODS.getNoInterestCount());
-            }
+        if (lastMinuteDWD != null && lastMinuteDWD.getHour().equals(result.getHour())) {
+            result.setCostDeviationRateMinute(result.getCostDeviationRateHour() - lastMinuteDWD.getCostDeviationRateHour());
+            result.setCostMinute(result.getCostHour() - lastMinuteDWD.getCostHour());
+            result.setCompensationAmountMinute(result.getCompensationAmountHour() - lastMinuteDWD.getCompensationAmountHour());
+            result.setViewCountMinute(result.getViewCountHour() - lastMinuteDWD.getViewCountHour());
+            result.setThousandDisplayPriceMinute(result.getThousandDisplayPriceHour() - lastMinuteDWD.getThousandDisplayPriceHour());
+            result.setAvgViewPerUserMinute(result.getAvgViewPerUserHour() - lastMinuteDWD.getAvgViewPerUserHour());
+            result.setValidClickCountMinute(result.getValidClickCountHour() - lastMinuteDWD.getValidClickCountHour());
+            result.setCtrMinute(result.getCtrHour() - lastMinuteDWD.getCtrHour());
+            result.setCpcMinute(result.getCpcHour() - lastMinuteDWD.getCpcHour());
+            result.setValuableClickCountMinute(result.getValuableClickCountHour() - lastMinuteDWD.getValuableClickCountHour());
+            result.setValuableClickRateMinute(result.getValuableClickRateHour() - lastMinuteDWD.getValuableClickRateHour());
+            result.setValuableClickCostMinute(result.getValuableClickCostHour() - lastMinuteDWD.getValuableClickCostHour());
+            result.setConversionsCountMinute(result.getConversionsCountHour() - lastMinuteDWD.getConversionsCountHour());
+            result.setConversionsCostMinute(result.getConversionsCostHour() - lastMinuteDWD.getConversionsCostHour());
+            result.setConversionsRateMinute(result.getConversionsRateHour() - lastMinuteDWD.getConversionsRateHour());
+            result.setDeepConversionsCountMinute(result.getDeepConversionsCountHour() - lastMinuteDWD.getDeepConversionsCountHour());
+            result.setDeepConversionsCostMinute(result.getDeepConversionsCostHour() - lastMinuteDWD.getDeepConversionsCostHour());
+            result.setDeepConversionsRateMinute(result.getDeepConversionsRateHour() - lastMinuteDWD.getDeepConversionsRateHour());
+            result.setOrderCountMinute(result.getOrderCountHour() - lastMinuteDWD.getOrderCountHour());
+            result.setFirstDayOrderCountMinute(result.getFirstDayOrderCountHour() - lastMinuteDWD.getFirstDayOrderCountHour());
+            result.setWebOrderCostMinute(result.getWebOrderCostHour() - lastMinuteDWD.getWebOrderCostHour());
+            result.setOrderRateMinute(result.getOrderRateHour() - lastMinuteDWD.getOrderRateHour());
+            result.setOrderAmountMinute(result.getOrderAmountHour() - lastMinuteDWD.getOrderAmountHour());
+            result.setFirstDayOrderAmountMinute(result.getFirstDayOrderAmountHour() - lastMinuteDWD.getFirstDayOrderAmountHour());
+            result.setOrderUnitPriceMinute(result.getOrderUnitPriceHour() - lastMinuteDWD.getOrderUnitPriceHour());
+            result.setOrderRoiMinute(result.getOrderRoiHour() - lastMinuteDWD.getOrderRoiHour());
+            result.setSignInCountMinute(result.getSignInCountHour() - lastMinuteDWD.getSignInCountHour());
+            result.setScanFollowCountMinute(result.getScanFollowCountHour() - lastMinuteDWD.getScanFollowCountHour());
+            result.setWechatAppRegisterUvMinute(result.getWechatAppRegisterUvHour() - lastMinuteDWD.getWechatAppRegisterUvHour());
+            result.setWechatMinigameRegisterCostMinute(result.getWechatMinigameRegisterCostHour() - lastMinuteDWD.getWechatMinigameRegisterCostHour());
+            result.setWechatMinigameRegisterRateMinute(result.getWechatMinigameRegisterRateHour() - lastMinuteDWD.getWechatMinigameRegisterRateHour());
+            result.setWechatMinigameArpuMinute(result.getWechatMinigameArpuHour() - lastMinuteDWD.getWechatMinigameArpuHour());
+            result.setWechatMinigameRetentionCountMinute(result.getWechatMinigameRetentionCountHour() - lastMinuteDWD.getWechatMinigameRetentionCountHour());
+            result.setWechatMinigameCheckoutCountMinute(result.getWechatMinigameCheckoutCountHour() - lastMinuteDWD.getWechatMinigameCheckoutCountHour());
+            result.setWechatMinigameCheckoutAmountMinute(result.getWechatMinigameCheckoutAmountHour() - lastMinuteDWD.getWechatMinigameCheckoutAmountHour());
+            result.setOfficialAccountFollowCountMinute(result.getOfficialAccountFollowCountHour() - lastMinuteDWD.getOfficialAccountFollowCountHour());
+            result.setOfficialAccountFollowRateMinute(result.getOfficialAccountFollowRateHour() - lastMinuteDWD.getOfficialAccountFollowRateHour());
+            result.setOfficialAccountRegisterUserCountMinute(result.getOfficialAccountRegisterUserCountHour() - lastMinuteDWD.getOfficialAccountRegisterUserCountHour());
+            result.setOfficialAccountRegisterRateMinute(result.getOfficialAccountRegisterRateHour() - lastMinuteDWD.getOfficialAccountRegisterRateHour());
+            result.setOfficialAccountRegisterCostMinute(result.getOfficialAccountRegisterCostHour() - lastMinuteDWD.getOfficialAccountRegisterCostHour());
+            result.setOfficialAccountRegisterAmountMinute(result.getOfficialAccountRegisterAmountHour() - lastMinuteDWD.getOfficialAccountRegisterAmountHour());
+            result.setOfficialAccountRegisterRoiMinute(result.getOfficialAccountRegisterRoiHour() - lastMinuteDWD.getOfficialAccountRegisterRoiHour());
+            result.setOfficialAccountApplyCountMinute(result.getOfficialAccountApplyCountHour() - lastMinuteDWD.getOfficialAccountApplyCountHour());
+            result.setOfficialAccountApplyUserCountMinute(result.getOfficialAccountApplyUserCountHour() - lastMinuteDWD.getOfficialAccountApplyUserCountHour());
+            result.setOfficialAccountApplyRateMinute(result.getOfficialAccountApplyRateHour() - lastMinuteDWD.getOfficialAccountApplyRateHour());
+            result.setOfficialAccountApplyCostMinute(result.getOfficialAccountApplyCostHour() - lastMinuteDWD.getOfficialAccountApplyCostHour());
+            result.setOfficialAccountApplyAmountMinute(result.getOfficialAccountApplyAmountHour() - lastMinuteDWD.getOfficialAccountApplyAmountHour());
+            result.setOfficialAccountApplyRoiMinute(result.getOfficialAccountApplyRoiHour() - lastMinuteDWD.getOfficialAccountApplyRoiHour());
+            result.setOfficialAccountOrderCountMinute(result.getOfficialAccountOrderCountHour() - lastMinuteDWD.getOfficialAccountOrderCountHour());
+            result.setOfficialAccountFirstDayOrderCountMinute(result.getOfficialAccountFirstDayOrderCountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderCountHour());
+            result.setOfficialAccountOrderUserCountMinute(result.getOfficialAccountOrderUserCountHour() - lastMinuteDWD.getOfficialAccountOrderUserCountHour());
+            result.setOfficialAccountOrderRateMinute(result.getOfficialAccountOrderRateHour() - lastMinuteDWD.getOfficialAccountOrderRateHour());
+            result.setOfficialAccountOrderCostMinute(result.getOfficialAccountOrderCostHour() - lastMinuteDWD.getOfficialAccountOrderCostHour());
+            result.setOfficialAccountOrderAmountMinute(result.getOfficialAccountOrderAmountHour() - lastMinuteDWD.getOfficialAccountOrderAmountHour());
+            result.setOfficialAccountFirstDayOrderAmountMinute(result.getOfficialAccountFirstDayOrderAmountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderAmountHour());
+            result.setOfficialAccountOrderRoiMinute(result.getOfficialAccountOrderRoiHour() - lastMinuteDWD.getOfficialAccountOrderRoiHour());
+            result.setOfficialAccountConsultCountMinute(result.getOfficialAccountConsultCountHour() - lastMinuteDWD.getOfficialAccountConsultCountHour());
+            result.setOfficialAccountReaderCountMinute(result.getOfficialAccountReaderCountHour() - lastMinuteDWD.getOfficialAccountReaderCountHour());
+            result.setOfficialAccountCreditApplyUserCountMinute(result.getOfficialAccountCreditApplyUserCountHour() - lastMinuteDWD.getOfficialAccountCreditApplyUserCountHour());
+            result.setOfficialAccountCreditUserCountMinute(result.getOfficialAccountCreditUserCountHour() - lastMinuteDWD.getOfficialAccountCreditUserCountHour());
+            result.setForwardCountMinute(result.getForwardCountHour() - lastMinuteDWD.getForwardCountHour());
+            result.setForwardUserCountMinute(result.getForwardUserCountHour() - lastMinuteDWD.getForwardUserCountHour());
+            result.setNoInterestCountMinute(result.getNoInterestCountHour() - lastMinuteDWD.getNoInterestCountHour());
+        } else {
+            result.setCostDeviationRateMinute(statODS.getCostDeviationRate());
+            result.setCostMinute(statODS.getCost());
+            result.setCompensationAmountMinute(statODS.getCompensationAmount());
+            result.setViewCountMinute(statODS.getViewCount());
+            result.setThousandDisplayPriceMinute(statODS.getThousandDisplayPrice());
+            result.setAvgViewPerUserMinute(statODS.getAvgViewPerUser());
+            result.setValidClickCountMinute(statODS.getValidClickCount());
+            result.setCtrMinute(statODS.getCtr());
+            result.setCpcMinute(statODS.getCpc());
+            result.setValuableClickCountMinute(statODS.getValuableClickCount());
+            result.setValuableClickRateMinute(statODS.getValuableClickRate());
+            result.setValuableClickCostMinute(statODS.getValuableClickCost());
+            result.setConversionsCountMinute(statODS.getConversionsCount());
+            result.setConversionsCostMinute(statODS.getConversionsCost());
+            result.setConversionsRateMinute(statODS.getConversionsRate());
+            result.setDeepConversionsCountMinute(statODS.getDeepConversionsCount());
+            result.setDeepConversionsCostMinute(statODS.getDeepConversionsCost());
+            result.setDeepConversionsRateMinute(statODS.getDeepConversionsRate());
+            result.setOrderCountMinute(statODS.getOrderCount());
+            result.setFirstDayOrderCountMinute(statODS.getFirstDayOrderCount());
+            result.setWebOrderCostMinute(statODS.getWebOrderCost());
+            result.setOrderRateMinute(statODS.getOrderRate());
+            result.setOrderAmountMinute(statODS.getOrderAmount());
+            result.setFirstDayOrderAmountMinute(statODS.getFirstDayOrderAmount());
+            result.setOrderUnitPriceMinute(statODS.getOrderUnitPrice());
+            result.setOrderRoiMinute(statODS.getOrderRoi());
+            result.setSignInCountMinute(statODS.getSignInCount());
+            result.setScanFollowCountMinute(statODS.getScanFollowCount());
+            result.setWechatAppRegisterUvMinute(statODS.getWechatAppRegisterUv());
+            result.setWechatMinigameRegisterCostMinute(statODS.getWechatMinigameRegisterCost());
+            result.setWechatMinigameRegisterRateMinute(statODS.getWechatMinigameRegisterRate());
+            result.setWechatMinigameArpuMinute(statODS.getWechatMinigameArpu());
+            result.setWechatMinigameRetentionCountMinute(statODS.getWechatMinigameRetentionCount());
+            result.setWechatMinigameCheckoutCountMinute(statODS.getWechatMinigameCheckoutCount());
+            result.setWechatMinigameCheckoutAmountMinute(statODS.getWechatMinigameCheckoutAmount());
+            result.setOfficialAccountFollowCountMinute(statODS.getOfficialAccountFollowCount());
+            result.setOfficialAccountFollowRateMinute(statODS.getOfficialAccountFollowRate());
+            result.setOfficialAccountRegisterUserCountMinute(statODS.getOfficialAccountRegisterUserCount());
+            result.setOfficialAccountRegisterRateMinute(statODS.getOfficialAccountRegisterRate());
+            result.setOfficialAccountRegisterCostMinute(statODS.getOfficialAccountRegisterCost());
+            result.setOfficialAccountRegisterAmountMinute(statODS.getOfficialAccountRegisterAmount());
+            result.setOfficialAccountRegisterRoiMinute(statODS.getOfficialAccountRegisterRoi());
+            result.setOfficialAccountApplyCountMinute(statODS.getOfficialAccountApplyCount());
+            result.setOfficialAccountApplyUserCountMinute(statODS.getOfficialAccountApplyUserCount());
+            result.setOfficialAccountApplyRateMinute(statODS.getOfficialAccountApplyRate());
+            result.setOfficialAccountApplyCostMinute(statODS.getOfficialAccountApplyCost());
+            result.setOfficialAccountApplyAmountMinute(statODS.getOfficialAccountApplyAmount());
+            result.setOfficialAccountApplyRoiMinute(statODS.getOfficialAccountApplyRoi());
+            result.setOfficialAccountOrderCountMinute(statODS.getOfficialAccountOrderCount());
+            result.setOfficialAccountFirstDayOrderCountMinute(statODS.getOfficialAccountFirstDayOrderCount());
+            result.setOfficialAccountOrderUserCountMinute(statODS.getOfficialAccountOrderUserCount());
+            result.setOfficialAccountOrderRateMinute(statODS.getOfficialAccountOrderRate());
+            result.setOfficialAccountOrderCostMinute(statODS.getOfficialAccountOrderCost());
+            result.setOfficialAccountOrderAmountMinute(statODS.getOfficialAccountOrderAmount());
+            result.setOfficialAccountFirstDayOrderAmountMinute(statODS.getOfficialAccountFirstDayOrderAmount());
+            result.setOfficialAccountOrderRoiMinute(statODS.getOfficialAccountOrderRoi());
+            result.setOfficialAccountConsultCountMinute(statODS.getOfficialAccountConsultCount());
+            result.setOfficialAccountReaderCountMinute(statODS.getOfficialAccountReaderCount());
+            result.setOfficialAccountCreditApplyUserCountMinute(statODS.getOfficialAccountCreditApplyUserCount());
+            result.setOfficialAccountCreditUserCountMinute(statODS.getOfficialAccountCreditUserCount());
+            result.setForwardCountMinute(statODS.getForwardCount());
+            result.setForwardUserCountMinute(statODS.getForwardUserCount());
+            result.setNoInterestCountMinute(statODS.getNoInterestCount());
         }
         return result;
     }

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/pojo/entity/PlanStatOfHourDWD.java

@@ -1215,6 +1215,12 @@ public class PlanStatOfHourDWD {
     @SerializedName("no_interest_count_hour")
     private Long noInterestCountHour;
 
+    public static PlanStatOfHourDWD byMinuteDWD(PlanStatOfMinuteDWD minuteDWD) {
+        PlanStatOfHourDWD result = new PlanStatOfHourDWD();
+        BeanUtils.copyProperties(minuteDWD, result);
+        return result;
+    }
+
     public static PlanStatOfHourDWD reduce(PlanStatOfHourDWD value1, AdDataOfHourODS value2, long createTime) {
         PlanStatOfHourDWD result = new PlanStatOfHourDWD();
         BeanUtils.copyProperties(value2, result);

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/pojo/entity/PlanStatOfMinuteDWD.java

@@ -1862,42 +1862,42 @@ public class PlanStatOfMinuteDWD {
             result.getAdIds().addAll(yesterdayMinuteDWD.getAdIds());
             PlanUtil.reduceAdGroup(result.getAdGroupMap(), yesterdayMinuteDWD.getAdGroupMap());
 
-            result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + yesterdayMinuteDWD.getCostDeviationRateTotal());
-            result.setCostTotal(result.getCostTotal() + yesterdayMinuteDWD.getCostTotal());
-            result.setCompensationAmountTotal(result.getCompensationAmountTotal() + yesterdayMinuteDWD.getCompensationAmountTotal());
-            result.setViewCountTotal(result.getViewCountTotal() + yesterdayMinuteDWD.getCompensationAmountTotal());
-            result.setValidClickCountTotal(result.getValidClickCountTotal() + yesterdayMinuteDWD.getValidClickCountTotal());
-            result.setValuableClickCountTotal(result.getValuableClickCountTotal() + yesterdayMinuteDWD.getValuableClickCountTotal());
-            result.setConversionsCountTotal(result.getConversionsCountTotal() + yesterdayMinuteDWD.getConversionsCountTotal());
-            result.setDeepConversionsCountTotal(result.getDeepConversionsCountTotal() + yesterdayMinuteDWD.getDeepConversionsCountTotal());
-            result.setOrderCountTotal(result.getOrderCountTotal() + yesterdayMinuteDWD.getOrderCountTotal());
-            result.setFirstDayOrderCountTotal(result.getFirstDayOrderCountTotal() + yesterdayMinuteDWD.getFirstDayOrderCountTotal());
-            result.setOrderAmountTotal(result.getOrderAmountTotal() + yesterdayMinuteDWD.getOrderAmountTotal());
-            result.setFirstDayOrderAmountTotal(result.getFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getFirstDayOrderAmountTotal());
-            result.setSignInCountTotal(result.getSignInCountTotal() + yesterdayMinuteDWD.getSignInCountTotal());
-            result.setScanFollowCountTotal(result.getScanFollowCountTotal() + yesterdayMinuteDWD.getScanFollowCountTotal());
-            result.setWechatAppRegisterUvTotal(result.getWechatAppRegisterUvTotal() + yesterdayMinuteDWD.getWechatAppRegisterUvTotal());
-            result.setWechatMinigameRetentionCountTotal(result.getWechatMinigameRetentionCountTotal() + yesterdayMinuteDWD.getWechatMinigameRetentionCountTotal());
-            result.setWechatMinigameCheckoutCountTotal(result.getWechatMinigameCheckoutCountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutCountTotal());
-            result.setWechatMinigameCheckoutAmountTotal(result.getWechatMinigameCheckoutAmountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutAmountTotal());
-            result.setOfficialAccountFollowCountTotal(result.getOfficialAccountFollowCountTotal() + yesterdayMinuteDWD.getOfficialAccountFollowCountTotal());
-            result.setOfficialAccountRegisterUserCountTotal(result.getOfficialAccountRegisterUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterUserCountTotal());
-            result.setOfficialAccountRegisterAmountTotal(result.getOfficialAccountRegisterAmountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterAmountTotal());
-            result.setOfficialAccountApplyCountTotal(result.getOfficialAccountApplyCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyCountTotal());
-            result.setOfficialAccountApplyUserCountTotal(result.getOfficialAccountApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyUserCountTotal());
-            result.setOfficialAccountApplyAmountTotal(result.getOfficialAccountApplyAmountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyAmountTotal());
-            result.setOfficialAccountOrderCountTotal(result.getOfficialAccountOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderCountTotal());
-            result.setOfficialAccountFirstDayOrderCountTotal(result.getOfficialAccountFirstDayOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderCountTotal());
-            result.setOfficialAccountOrderUserCountTotal(result.getOfficialAccountOrderUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderUserCountTotal());
-            result.setOfficialAccountOrderAmountTotal(result.getOfficialAccountOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderAmountTotal());
-            result.setOfficialAccountFirstDayOrderAmountTotal(result.getOfficialAccountFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderAmountTotal());
-            result.setOfficialAccountConsultCountTotal(result.getOfficialAccountConsultCountTotal() + yesterdayMinuteDWD.getOfficialAccountConsultCountTotal());
-            result.setOfficialAccountReaderCountTotal(result.getOfficialAccountReaderCountTotal() + yesterdayMinuteDWD.getOfficialAccountReaderCountTotal());
-            result.setOfficialAccountCreditApplyUserCountTotal(result.getOfficialAccountCreditApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditApplyUserCountTotal());
-            result.setOfficialAccountCreditUserCountTotal(result.getOfficialAccountCreditUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditUserCountTotal());
-            result.setForwardCountTotal(result.getForwardCountTotal() + yesterdayMinuteDWD.getForwardCountTotal());
-            result.setForwardUserCountTotal(result.getForwardUserCountTotal() + yesterdayMinuteDWD.getForwardUserCountTotal());
-            result.setNoInterestCountTotal(result.getNoInterestCountTotal() + yesterdayMinuteDWD.getNoInterestCountTotal());
+            result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + yesterdayMinuteDWD.getCostDeviationRateDay());
+            result.setCostTotal(result.getCostTotal() + yesterdayMinuteDWD.getCostDay());
+            result.setCompensationAmountTotal(result.getCompensationAmountTotal() + yesterdayMinuteDWD.getCompensationAmountDay());
+            result.setViewCountTotal(result.getViewCountTotal() + yesterdayMinuteDWD.getCompensationAmountDay());
+            result.setValidClickCountTotal(result.getValidClickCountTotal() + yesterdayMinuteDWD.getValidClickCountDay());
+            result.setValuableClickCountTotal(result.getValuableClickCountTotal() + yesterdayMinuteDWD.getValuableClickCountDay());
+            result.setConversionsCountTotal(result.getConversionsCountTotal() + yesterdayMinuteDWD.getConversionsCountDay());
+            result.setDeepConversionsCountTotal(result.getDeepConversionsCountTotal() + yesterdayMinuteDWD.getDeepConversionsCountDay());
+            result.setOrderCountTotal(result.getOrderCountTotal() + yesterdayMinuteDWD.getOrderCountDay());
+            result.setFirstDayOrderCountTotal(result.getFirstDayOrderCountTotal() + yesterdayMinuteDWD.getFirstDayOrderCountDay());
+            result.setOrderAmountTotal(result.getOrderAmountTotal() + yesterdayMinuteDWD.getOrderAmountDay());
+            result.setFirstDayOrderAmountTotal(result.getFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getFirstDayOrderAmountDay());
+            result.setSignInCountTotal(result.getSignInCountTotal() + yesterdayMinuteDWD.getSignInCountDay());
+            result.setScanFollowCountTotal(result.getScanFollowCountTotal() + yesterdayMinuteDWD.getScanFollowCountDay());
+            result.setWechatAppRegisterUvTotal(result.getWechatAppRegisterUvTotal() + yesterdayMinuteDWD.getWechatAppRegisterUvDay());
+            result.setWechatMinigameRetentionCountTotal(result.getWechatMinigameRetentionCountTotal() + yesterdayMinuteDWD.getWechatMinigameRetentionCountDay());
+            result.setWechatMinigameCheckoutCountTotal(result.getWechatMinigameCheckoutCountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutCountDay());
+            result.setWechatMinigameCheckoutAmountTotal(result.getWechatMinigameCheckoutAmountTotal() + yesterdayMinuteDWD.getWechatMinigameCheckoutAmountDay());
+            result.setOfficialAccountFollowCountTotal(result.getOfficialAccountFollowCountTotal() + yesterdayMinuteDWD.getOfficialAccountFollowCountDay());
+            result.setOfficialAccountRegisterUserCountTotal(result.getOfficialAccountRegisterUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterUserCountDay());
+            result.setOfficialAccountRegisterAmountTotal(result.getOfficialAccountRegisterAmountTotal() + yesterdayMinuteDWD.getOfficialAccountRegisterAmountDay());
+            result.setOfficialAccountApplyCountTotal(result.getOfficialAccountApplyCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyCountDay());
+            result.setOfficialAccountApplyUserCountTotal(result.getOfficialAccountApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyUserCountDay());
+            result.setOfficialAccountApplyAmountTotal(result.getOfficialAccountApplyAmountTotal() + yesterdayMinuteDWD.getOfficialAccountApplyAmountDay());
+            result.setOfficialAccountOrderCountTotal(result.getOfficialAccountOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderCountDay());
+            result.setOfficialAccountFirstDayOrderCountTotal(result.getOfficialAccountFirstDayOrderCountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderCountDay());
+            result.setOfficialAccountOrderUserCountTotal(result.getOfficialAccountOrderUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderUserCountDay());
+            result.setOfficialAccountOrderAmountTotal(result.getOfficialAccountOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountOrderAmountDay());
+            result.setOfficialAccountFirstDayOrderAmountTotal(result.getOfficialAccountFirstDayOrderAmountTotal() + yesterdayMinuteDWD.getOfficialAccountFirstDayOrderAmountDay());
+            result.setOfficialAccountConsultCountTotal(result.getOfficialAccountConsultCountTotal() + yesterdayMinuteDWD.getOfficialAccountConsultCountDay());
+            result.setOfficialAccountReaderCountTotal(result.getOfficialAccountReaderCountTotal() + yesterdayMinuteDWD.getOfficialAccountReaderCountDay());
+            result.setOfficialAccountCreditApplyUserCountTotal(result.getOfficialAccountCreditApplyUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditApplyUserCountDay());
+            result.setOfficialAccountCreditUserCountTotal(result.getOfficialAccountCreditUserCountTotal() + yesterdayMinuteDWD.getOfficialAccountCreditUserCountDay());
+            result.setForwardCountTotal(result.getForwardCountTotal() + yesterdayMinuteDWD.getForwardCountDay());
+            result.setForwardUserCountTotal(result.getForwardUserCountTotal() + yesterdayMinuteDWD.getForwardUserCountDay());
+            result.setNoInterestCountTotal(result.getNoInterestCountTotal() + yesterdayMinuteDWD.getNoInterestCountDay());
         }
         result.setCostDeviationRateTotal(result.getCostDeviationRateTotal() + result.getCostDeviationRateDay());
         result.setCostTotal(result.getCostTotal() + result.getCostDay());
@@ -1989,136 +1989,134 @@ public class PlanStatOfMinuteDWD {
         result.setOfficialAccountOrderRoiAll(result.getCostTotal() == 0 ? 0 : result.getOfficialAccountOrderAmountTotal() / result.getCostTotal());
 
         // 上一次聚合的 5分钟数据
-        if (lastMinuteDWD != null) {
-            if (lastMinuteDWD.getHour().equals(result.getHour())) {
-                result.setCostDeviationRateMinute(result.getCostDeviationRateHour() - lastMinuteDWD.getCostDeviationRateHour());
-                result.setCostMinute(result.getCostHour() - lastMinuteDWD.getCostHour());
-                result.setCompensationAmountMinute(result.getCompensationAmountHour() - lastMinuteDWD.getCompensationAmountHour());
-                result.setViewCountMinute(result.getViewCountHour() - lastMinuteDWD.getViewCountHour());
-                result.setThousandDisplayPriceMinute(result.getThousandDisplayPriceHour() - lastMinuteDWD.getThousandDisplayPriceHour());
-                result.setAvgViewPerUserMinute(result.getAvgViewPerUserHour() - lastMinuteDWD.getAvgViewPerUserHour());
-                result.setValidClickCountMinute(result.getValidClickCountHour() - lastMinuteDWD.getValidClickCountHour());
-                result.setCtrMinute(result.getCtrHour() - lastMinuteDWD.getCtrHour());
-                result.setCpcMinute(result.getCpcHour() - lastMinuteDWD.getCpcHour());
-                result.setValuableClickCountMinute(result.getValuableClickCountHour() - lastMinuteDWD.getValuableClickCountHour());
-                result.setValuableClickRateMinute(result.getValuableClickRateHour() - lastMinuteDWD.getValuableClickRateHour());
-                result.setValuableClickCostMinute(result.getValuableClickCostHour() - lastMinuteDWD.getValuableClickCostHour());
-                result.setConversionsCountMinute(result.getConversionsCountHour() - lastMinuteDWD.getConversionsCountHour());
-                result.setConversionsCostMinute(result.getConversionsCostHour() - lastMinuteDWD.getConversionsCostHour());
-                result.setConversionsRateMinute(result.getConversionsRateHour() - lastMinuteDWD.getConversionsRateHour());
-                result.setDeepConversionsCountMinute(result.getDeepConversionsCountHour() - lastMinuteDWD.getDeepConversionsCountHour());
-                result.setDeepConversionsCostMinute(result.getDeepConversionsCostHour() - lastMinuteDWD.getDeepConversionsCostHour());
-                result.setDeepConversionsRateMinute(result.getDeepConversionsRateHour() - lastMinuteDWD.getDeepConversionsRateHour());
-                result.setOrderCountMinute(result.getOrderCountHour() - lastMinuteDWD.getOrderCountHour());
-                result.setFirstDayOrderCountMinute(result.getFirstDayOrderCountHour() - lastMinuteDWD.getFirstDayOrderCountHour());
-                result.setWebOrderCostMinute(result.getWebOrderCostHour() - lastMinuteDWD.getWebOrderCostHour());
-                result.setOrderRateMinute(result.getOrderRateHour() - lastMinuteDWD.getOrderRateHour());
-                result.setOrderAmountMinute(result.getOrderAmountHour() - lastMinuteDWD.getOrderAmountHour());
-                result.setFirstDayOrderAmountMinute(result.getFirstDayOrderAmountHour() - lastMinuteDWD.getFirstDayOrderAmountHour());
-                result.setOrderUnitPriceMinute(result.getOrderUnitPriceHour() - lastMinuteDWD.getOrderUnitPriceHour());
-                result.setOrderRoiMinute(result.getOrderRoiHour() - lastMinuteDWD.getOrderRoiHour());
-                result.setSignInCountMinute(result.getSignInCountHour() - lastMinuteDWD.getSignInCountHour());
-                result.setScanFollowCountMinute(result.getScanFollowCountHour() - lastMinuteDWD.getScanFollowCountHour());
-                result.setWechatAppRegisterUvMinute(result.getWechatAppRegisterUvHour() - lastMinuteDWD.getWechatAppRegisterUvHour());
-                result.setWechatMinigameRegisterCostMinute(result.getWechatMinigameRegisterCostHour() - lastMinuteDWD.getWechatMinigameRegisterCostHour());
-                result.setWechatMinigameRegisterRateMinute(result.getWechatMinigameRegisterRateHour() - lastMinuteDWD.getWechatMinigameRegisterRateHour());
-                result.setWechatMinigameArpuMinute(result.getWechatMinigameArpuHour() - lastMinuteDWD.getWechatMinigameArpuHour());
-                result.setWechatMinigameRetentionCountMinute(result.getWechatMinigameRetentionCountHour() - lastMinuteDWD.getWechatMinigameRetentionCountHour());
-                result.setWechatMinigameCheckoutCountMinute(result.getWechatMinigameCheckoutCountHour() - lastMinuteDWD.getWechatMinigameCheckoutCountHour());
-                result.setWechatMinigameCheckoutAmountMinute(result.getWechatMinigameCheckoutAmountHour() - lastMinuteDWD.getWechatMinigameCheckoutAmountHour());
-                result.setOfficialAccountFollowCountMinute(result.getOfficialAccountFollowCountHour() - lastMinuteDWD.getOfficialAccountFollowCountHour());
-                result.setOfficialAccountFollowRateMinute(result.getOfficialAccountFollowRateHour() - lastMinuteDWD.getOfficialAccountFollowRateHour());
-                result.setOfficialAccountRegisterUserCountMinute(result.getOfficialAccountRegisterUserCountHour() - lastMinuteDWD.getOfficialAccountRegisterUserCountHour());
-                result.setOfficialAccountRegisterRateMinute(result.getOfficialAccountRegisterRateHour() - lastMinuteDWD.getOfficialAccountRegisterRateHour());
-                result.setOfficialAccountRegisterCostMinute(result.getOfficialAccountRegisterCostHour() - lastMinuteDWD.getOfficialAccountRegisterCostHour());
-                result.setOfficialAccountRegisterAmountMinute(result.getOfficialAccountRegisterAmountHour() - lastMinuteDWD.getOfficialAccountRegisterAmountHour());
-                result.setOfficialAccountRegisterRoiMinute(result.getOfficialAccountRegisterRoiHour() - lastMinuteDWD.getOfficialAccountRegisterRoiHour());
-                result.setOfficialAccountApplyCountMinute(result.getOfficialAccountApplyCountHour() - lastMinuteDWD.getOfficialAccountApplyCountHour());
-                result.setOfficialAccountApplyUserCountMinute(result.getOfficialAccountApplyUserCountHour() - lastMinuteDWD.getOfficialAccountApplyUserCountHour());
-                result.setOfficialAccountApplyRateMinute(result.getOfficialAccountApplyRateHour() - lastMinuteDWD.getOfficialAccountApplyRateHour());
-                result.setOfficialAccountApplyCostMinute(result.getOfficialAccountApplyCostHour() - lastMinuteDWD.getOfficialAccountApplyCostHour());
-                result.setOfficialAccountApplyAmountMinute(result.getOfficialAccountApplyAmountHour() - lastMinuteDWD.getOfficialAccountApplyAmountHour());
-                result.setOfficialAccountApplyRoiMinute(result.getOfficialAccountApplyRoiHour() - lastMinuteDWD.getOfficialAccountApplyRoiHour());
-                result.setOfficialAccountOrderCountMinute(result.getOfficialAccountOrderCountHour() - lastMinuteDWD.getOfficialAccountOrderCountHour());
-                result.setOfficialAccountFirstDayOrderCountMinute(result.getOfficialAccountFirstDayOrderCountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderCountHour());
-                result.setOfficialAccountOrderUserCountMinute(result.getOfficialAccountOrderUserCountHour() - lastMinuteDWD.getOfficialAccountOrderUserCountHour());
-                result.setOfficialAccountOrderRateMinute(result.getOfficialAccountOrderRateHour() - lastMinuteDWD.getOfficialAccountOrderRateHour());
-                result.setOfficialAccountOrderCostMinute(result.getOfficialAccountOrderCostHour() - lastMinuteDWD.getOfficialAccountOrderCostHour());
-                result.setOfficialAccountOrderAmountMinute(result.getOfficialAccountOrderAmountHour() - lastMinuteDWD.getOfficialAccountOrderAmountHour());
-                result.setOfficialAccountFirstDayOrderAmountMinute(result.getOfficialAccountFirstDayOrderAmountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderAmountHour());
-                result.setOfficialAccountOrderRoiMinute(result.getOfficialAccountOrderRoiHour() - lastMinuteDWD.getOfficialAccountOrderRoiHour());
-                result.setOfficialAccountConsultCountMinute(result.getOfficialAccountConsultCountHour() - lastMinuteDWD.getOfficialAccountConsultCountHour());
-                result.setOfficialAccountReaderCountMinute(result.getOfficialAccountReaderCountHour() - lastMinuteDWD.getOfficialAccountReaderCountHour());
-                result.setOfficialAccountCreditApplyUserCountMinute(result.getOfficialAccountCreditApplyUserCountHour() - lastMinuteDWD.getOfficialAccountCreditApplyUserCountHour());
-                result.setOfficialAccountCreditUserCountMinute(result.getOfficialAccountCreditUserCountHour() - lastMinuteDWD.getOfficialAccountCreditUserCountHour());
-                result.setForwardCountMinute(result.getForwardCountHour() - lastMinuteDWD.getForwardCountHour());
-                result.setForwardUserCountMinute(result.getForwardUserCountHour() - lastMinuteDWD.getForwardUserCountHour());
-                result.setNoInterestCountMinute(result.getNoInterestCountHour() - lastMinuteDWD.getNoInterestCountHour());
-            } else {
-                result.setCostDeviationRateMinute(statODS.getCostDeviationRate());
-                result.setCostMinute(statODS.getCost());
-                result.setCompensationAmountMinute(statODS.getCompensationAmount());
-                result.setViewCountMinute(statODS.getViewCount());
-                result.setThousandDisplayPriceMinute(statODS.getThousandDisplayPrice());
-                result.setAvgViewPerUserMinute(statODS.getAvgViewPerUser());
-                result.setValidClickCountMinute(statODS.getValidClickCount());
-                result.setCtrMinute(statODS.getCtr());
-                result.setCpcMinute(statODS.getCpc());
-                result.setValuableClickCountMinute(statODS.getValuableClickCount());
-                result.setValuableClickRateMinute(statODS.getValuableClickRate());
-                result.setValuableClickCostMinute(statODS.getValuableClickCost());
-                result.setConversionsCountMinute(statODS.getConversionsCount());
-                result.setConversionsCostMinute(statODS.getConversionsCost());
-                result.setConversionsRateMinute(statODS.getConversionsRate());
-                result.setDeepConversionsCountMinute(statODS.getDeepConversionsCount());
-                result.setDeepConversionsCostMinute(statODS.getDeepConversionsCost());
-                result.setDeepConversionsRateMinute(statODS.getDeepConversionsRate());
-                result.setOrderCountMinute(statODS.getOrderCount());
-                result.setFirstDayOrderCountMinute(statODS.getFirstDayOrderCount());
-                result.setWebOrderCostMinute(statODS.getWebOrderCost());
-                result.setOrderRateMinute(statODS.getOrderRate());
-                result.setOrderAmountMinute(statODS.getOrderAmount());
-                result.setFirstDayOrderAmountMinute(statODS.getFirstDayOrderAmount());
-                result.setOrderUnitPriceMinute(statODS.getOrderUnitPrice());
-                result.setOrderRoiMinute(statODS.getOrderRoi());
-                result.setSignInCountMinute(statODS.getSignInCount());
-                result.setScanFollowCountMinute(statODS.getScanFollowCount());
-                result.setWechatAppRegisterUvMinute(statODS.getWechatAppRegisterUv());
-                result.setWechatMinigameRegisterCostMinute(statODS.getWechatMinigameRegisterCost());
-                result.setWechatMinigameRegisterRateMinute(statODS.getWechatMinigameRegisterRate());
-                result.setWechatMinigameArpuMinute(statODS.getWechatMinigameArpu());
-                result.setWechatMinigameRetentionCountMinute(statODS.getWechatMinigameRetentionCount());
-                result.setWechatMinigameCheckoutCountMinute(statODS.getWechatMinigameCheckoutCount());
-                result.setWechatMinigameCheckoutAmountMinute(statODS.getWechatMinigameCheckoutAmount());
-                result.setOfficialAccountFollowCountMinute(statODS.getOfficialAccountFollowCount());
-                result.setOfficialAccountFollowRateMinute(statODS.getOfficialAccountFollowRate());
-                result.setOfficialAccountRegisterUserCountMinute(statODS.getOfficialAccountRegisterUserCount());
-                result.setOfficialAccountRegisterRateMinute(statODS.getOfficialAccountRegisterRate());
-                result.setOfficialAccountRegisterCostMinute(statODS.getOfficialAccountRegisterCost());
-                result.setOfficialAccountRegisterAmountMinute(statODS.getOfficialAccountRegisterAmount());
-                result.setOfficialAccountRegisterRoiMinute(statODS.getOfficialAccountRegisterRoi());
-                result.setOfficialAccountApplyCountMinute(statODS.getOfficialAccountApplyCount());
-                result.setOfficialAccountApplyUserCountMinute(statODS.getOfficialAccountApplyUserCount());
-                result.setOfficialAccountApplyRateMinute(statODS.getOfficialAccountApplyRate());
-                result.setOfficialAccountApplyCostMinute(statODS.getOfficialAccountApplyCost());
-                result.setOfficialAccountApplyAmountMinute(statODS.getOfficialAccountApplyAmount());
-                result.setOfficialAccountApplyRoiMinute(statODS.getOfficialAccountApplyRoi());
-                result.setOfficialAccountOrderCountMinute(statODS.getOfficialAccountOrderCount());
-                result.setOfficialAccountFirstDayOrderCountMinute(statODS.getOfficialAccountFirstDayOrderCount());
-                result.setOfficialAccountOrderUserCountMinute(statODS.getOfficialAccountOrderUserCount());
-                result.setOfficialAccountOrderRateMinute(statODS.getOfficialAccountOrderRate());
-                result.setOfficialAccountOrderCostMinute(statODS.getOfficialAccountOrderCost());
-                result.setOfficialAccountOrderAmountMinute(statODS.getOfficialAccountOrderAmount());
-                result.setOfficialAccountFirstDayOrderAmountMinute(statODS.getOfficialAccountFirstDayOrderAmount());
-                result.setOfficialAccountOrderRoiMinute(statODS.getOfficialAccountOrderRoi());
-                result.setOfficialAccountConsultCountMinute(statODS.getOfficialAccountConsultCount());
-                result.setOfficialAccountReaderCountMinute(statODS.getOfficialAccountReaderCount());
-                result.setOfficialAccountCreditApplyUserCountMinute(statODS.getOfficialAccountCreditApplyUserCount());
-                result.setOfficialAccountCreditUserCountMinute(statODS.getOfficialAccountCreditUserCount());
-                result.setForwardCountMinute(statODS.getForwardCount());
-                result.setForwardUserCountMinute(statODS.getForwardUserCount());
-                result.setNoInterestCountMinute(statODS.getNoInterestCount());
-            }
+        if (lastMinuteDWD != null && lastMinuteDWD.getHour().equals(result.getHour())) {
+            result.setCostDeviationRateMinute(result.getCostDeviationRateHour() - lastMinuteDWD.getCostDeviationRateHour());
+            result.setCostMinute(result.getCostHour() - lastMinuteDWD.getCostHour());
+            result.setCompensationAmountMinute(result.getCompensationAmountHour() - lastMinuteDWD.getCompensationAmountHour());
+            result.setViewCountMinute(result.getViewCountHour() - lastMinuteDWD.getViewCountHour());
+            result.setThousandDisplayPriceMinute(result.getThousandDisplayPriceHour() - lastMinuteDWD.getThousandDisplayPriceHour());
+            result.setAvgViewPerUserMinute(result.getAvgViewPerUserHour() - lastMinuteDWD.getAvgViewPerUserHour());
+            result.setValidClickCountMinute(result.getValidClickCountHour() - lastMinuteDWD.getValidClickCountHour());
+            result.setCtrMinute(result.getCtrHour() - lastMinuteDWD.getCtrHour());
+            result.setCpcMinute(result.getCpcHour() - lastMinuteDWD.getCpcHour());
+            result.setValuableClickCountMinute(result.getValuableClickCountHour() - lastMinuteDWD.getValuableClickCountHour());
+            result.setValuableClickRateMinute(result.getValuableClickRateHour() - lastMinuteDWD.getValuableClickRateHour());
+            result.setValuableClickCostMinute(result.getValuableClickCostHour() - lastMinuteDWD.getValuableClickCostHour());
+            result.setConversionsCountMinute(result.getConversionsCountHour() - lastMinuteDWD.getConversionsCountHour());
+            result.setConversionsCostMinute(result.getConversionsCostHour() - lastMinuteDWD.getConversionsCostHour());
+            result.setConversionsRateMinute(result.getConversionsRateHour() - lastMinuteDWD.getConversionsRateHour());
+            result.setDeepConversionsCountMinute(result.getDeepConversionsCountHour() - lastMinuteDWD.getDeepConversionsCountHour());
+            result.setDeepConversionsCostMinute(result.getDeepConversionsCostHour() - lastMinuteDWD.getDeepConversionsCostHour());
+            result.setDeepConversionsRateMinute(result.getDeepConversionsRateHour() - lastMinuteDWD.getDeepConversionsRateHour());
+            result.setOrderCountMinute(result.getOrderCountHour() - lastMinuteDWD.getOrderCountHour());
+            result.setFirstDayOrderCountMinute(result.getFirstDayOrderCountHour() - lastMinuteDWD.getFirstDayOrderCountHour());
+            result.setWebOrderCostMinute(result.getWebOrderCostHour() - lastMinuteDWD.getWebOrderCostHour());
+            result.setOrderRateMinute(result.getOrderRateHour() - lastMinuteDWD.getOrderRateHour());
+            result.setOrderAmountMinute(result.getOrderAmountHour() - lastMinuteDWD.getOrderAmountHour());
+            result.setFirstDayOrderAmountMinute(result.getFirstDayOrderAmountHour() - lastMinuteDWD.getFirstDayOrderAmountHour());
+            result.setOrderUnitPriceMinute(result.getOrderUnitPriceHour() - lastMinuteDWD.getOrderUnitPriceHour());
+            result.setOrderRoiMinute(result.getOrderRoiHour() - lastMinuteDWD.getOrderRoiHour());
+            result.setSignInCountMinute(result.getSignInCountHour() - lastMinuteDWD.getSignInCountHour());
+            result.setScanFollowCountMinute(result.getScanFollowCountHour() - lastMinuteDWD.getScanFollowCountHour());
+            result.setWechatAppRegisterUvMinute(result.getWechatAppRegisterUvHour() - lastMinuteDWD.getWechatAppRegisterUvHour());
+            result.setWechatMinigameRegisterCostMinute(result.getWechatMinigameRegisterCostHour() - lastMinuteDWD.getWechatMinigameRegisterCostHour());
+            result.setWechatMinigameRegisterRateMinute(result.getWechatMinigameRegisterRateHour() - lastMinuteDWD.getWechatMinigameRegisterRateHour());
+            result.setWechatMinigameArpuMinute(result.getWechatMinigameArpuHour() - lastMinuteDWD.getWechatMinigameArpuHour());
+            result.setWechatMinigameRetentionCountMinute(result.getWechatMinigameRetentionCountHour() - lastMinuteDWD.getWechatMinigameRetentionCountHour());
+            result.setWechatMinigameCheckoutCountMinute(result.getWechatMinigameCheckoutCountHour() - lastMinuteDWD.getWechatMinigameCheckoutCountHour());
+            result.setWechatMinigameCheckoutAmountMinute(result.getWechatMinigameCheckoutAmountHour() - lastMinuteDWD.getWechatMinigameCheckoutAmountHour());
+            result.setOfficialAccountFollowCountMinute(result.getOfficialAccountFollowCountHour() - lastMinuteDWD.getOfficialAccountFollowCountHour());
+            result.setOfficialAccountFollowRateMinute(result.getOfficialAccountFollowRateHour() - lastMinuteDWD.getOfficialAccountFollowRateHour());
+            result.setOfficialAccountRegisterUserCountMinute(result.getOfficialAccountRegisterUserCountHour() - lastMinuteDWD.getOfficialAccountRegisterUserCountHour());
+            result.setOfficialAccountRegisterRateMinute(result.getOfficialAccountRegisterRateHour() - lastMinuteDWD.getOfficialAccountRegisterRateHour());
+            result.setOfficialAccountRegisterCostMinute(result.getOfficialAccountRegisterCostHour() - lastMinuteDWD.getOfficialAccountRegisterCostHour());
+            result.setOfficialAccountRegisterAmountMinute(result.getOfficialAccountRegisterAmountHour() - lastMinuteDWD.getOfficialAccountRegisterAmountHour());
+            result.setOfficialAccountRegisterRoiMinute(result.getOfficialAccountRegisterRoiHour() - lastMinuteDWD.getOfficialAccountRegisterRoiHour());
+            result.setOfficialAccountApplyCountMinute(result.getOfficialAccountApplyCountHour() - lastMinuteDWD.getOfficialAccountApplyCountHour());
+            result.setOfficialAccountApplyUserCountMinute(result.getOfficialAccountApplyUserCountHour() - lastMinuteDWD.getOfficialAccountApplyUserCountHour());
+            result.setOfficialAccountApplyRateMinute(result.getOfficialAccountApplyRateHour() - lastMinuteDWD.getOfficialAccountApplyRateHour());
+            result.setOfficialAccountApplyCostMinute(result.getOfficialAccountApplyCostHour() - lastMinuteDWD.getOfficialAccountApplyCostHour());
+            result.setOfficialAccountApplyAmountMinute(result.getOfficialAccountApplyAmountHour() - lastMinuteDWD.getOfficialAccountApplyAmountHour());
+            result.setOfficialAccountApplyRoiMinute(result.getOfficialAccountApplyRoiHour() - lastMinuteDWD.getOfficialAccountApplyRoiHour());
+            result.setOfficialAccountOrderCountMinute(result.getOfficialAccountOrderCountHour() - lastMinuteDWD.getOfficialAccountOrderCountHour());
+            result.setOfficialAccountFirstDayOrderCountMinute(result.getOfficialAccountFirstDayOrderCountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderCountHour());
+            result.setOfficialAccountOrderUserCountMinute(result.getOfficialAccountOrderUserCountHour() - lastMinuteDWD.getOfficialAccountOrderUserCountHour());
+            result.setOfficialAccountOrderRateMinute(result.getOfficialAccountOrderRateHour() - lastMinuteDWD.getOfficialAccountOrderRateHour());
+            result.setOfficialAccountOrderCostMinute(result.getOfficialAccountOrderCostHour() - lastMinuteDWD.getOfficialAccountOrderCostHour());
+            result.setOfficialAccountOrderAmountMinute(result.getOfficialAccountOrderAmountHour() - lastMinuteDWD.getOfficialAccountOrderAmountHour());
+            result.setOfficialAccountFirstDayOrderAmountMinute(result.getOfficialAccountFirstDayOrderAmountHour() - lastMinuteDWD.getOfficialAccountFirstDayOrderAmountHour());
+            result.setOfficialAccountOrderRoiMinute(result.getOfficialAccountOrderRoiHour() - lastMinuteDWD.getOfficialAccountOrderRoiHour());
+            result.setOfficialAccountConsultCountMinute(result.getOfficialAccountConsultCountHour() - lastMinuteDWD.getOfficialAccountConsultCountHour());
+            result.setOfficialAccountReaderCountMinute(result.getOfficialAccountReaderCountHour() - lastMinuteDWD.getOfficialAccountReaderCountHour());
+            result.setOfficialAccountCreditApplyUserCountMinute(result.getOfficialAccountCreditApplyUserCountHour() - lastMinuteDWD.getOfficialAccountCreditApplyUserCountHour());
+            result.setOfficialAccountCreditUserCountMinute(result.getOfficialAccountCreditUserCountHour() - lastMinuteDWD.getOfficialAccountCreditUserCountHour());
+            result.setForwardCountMinute(result.getForwardCountHour() - lastMinuteDWD.getForwardCountHour());
+            result.setForwardUserCountMinute(result.getForwardUserCountHour() - lastMinuteDWD.getForwardUserCountHour());
+            result.setNoInterestCountMinute(result.getNoInterestCountHour() - lastMinuteDWD.getNoInterestCountHour());
+        } else {
+            result.setCostDeviationRateMinute(statODS.getCostDeviationRate());
+            result.setCostMinute(statODS.getCost());
+            result.setCompensationAmountMinute(statODS.getCompensationAmount());
+            result.setViewCountMinute(statODS.getViewCount());
+            result.setThousandDisplayPriceMinute(statODS.getThousandDisplayPrice());
+            result.setAvgViewPerUserMinute(statODS.getAvgViewPerUser());
+            result.setValidClickCountMinute(statODS.getValidClickCount());
+            result.setCtrMinute(statODS.getCtr());
+            result.setCpcMinute(statODS.getCpc());
+            result.setValuableClickCountMinute(statODS.getValuableClickCount());
+            result.setValuableClickRateMinute(statODS.getValuableClickRate());
+            result.setValuableClickCostMinute(statODS.getValuableClickCost());
+            result.setConversionsCountMinute(statODS.getConversionsCount());
+            result.setConversionsCostMinute(statODS.getConversionsCost());
+            result.setConversionsRateMinute(statODS.getConversionsRate());
+            result.setDeepConversionsCountMinute(statODS.getDeepConversionsCount());
+            result.setDeepConversionsCostMinute(statODS.getDeepConversionsCost());
+            result.setDeepConversionsRateMinute(statODS.getDeepConversionsRate());
+            result.setOrderCountMinute(statODS.getOrderCount());
+            result.setFirstDayOrderCountMinute(statODS.getFirstDayOrderCount());
+            result.setWebOrderCostMinute(statODS.getWebOrderCost());
+            result.setOrderRateMinute(statODS.getOrderRate());
+            result.setOrderAmountMinute(statODS.getOrderAmount());
+            result.setFirstDayOrderAmountMinute(statODS.getFirstDayOrderAmount());
+            result.setOrderUnitPriceMinute(statODS.getOrderUnitPrice());
+            result.setOrderRoiMinute(statODS.getOrderRoi());
+            result.setSignInCountMinute(statODS.getSignInCount());
+            result.setScanFollowCountMinute(statODS.getScanFollowCount());
+            result.setWechatAppRegisterUvMinute(statODS.getWechatAppRegisterUv());
+            result.setWechatMinigameRegisterCostMinute(statODS.getWechatMinigameRegisterCost());
+            result.setWechatMinigameRegisterRateMinute(statODS.getWechatMinigameRegisterRate());
+            result.setWechatMinigameArpuMinute(statODS.getWechatMinigameArpu());
+            result.setWechatMinigameRetentionCountMinute(statODS.getWechatMinigameRetentionCount());
+            result.setWechatMinigameCheckoutCountMinute(statODS.getWechatMinigameCheckoutCount());
+            result.setWechatMinigameCheckoutAmountMinute(statODS.getWechatMinigameCheckoutAmount());
+            result.setOfficialAccountFollowCountMinute(statODS.getOfficialAccountFollowCount());
+            result.setOfficialAccountFollowRateMinute(statODS.getOfficialAccountFollowRate());
+            result.setOfficialAccountRegisterUserCountMinute(statODS.getOfficialAccountRegisterUserCount());
+            result.setOfficialAccountRegisterRateMinute(statODS.getOfficialAccountRegisterRate());
+            result.setOfficialAccountRegisterCostMinute(statODS.getOfficialAccountRegisterCost());
+            result.setOfficialAccountRegisterAmountMinute(statODS.getOfficialAccountRegisterAmount());
+            result.setOfficialAccountRegisterRoiMinute(statODS.getOfficialAccountRegisterRoi());
+            result.setOfficialAccountApplyCountMinute(statODS.getOfficialAccountApplyCount());
+            result.setOfficialAccountApplyUserCountMinute(statODS.getOfficialAccountApplyUserCount());
+            result.setOfficialAccountApplyRateMinute(statODS.getOfficialAccountApplyRate());
+            result.setOfficialAccountApplyCostMinute(statODS.getOfficialAccountApplyCost());
+            result.setOfficialAccountApplyAmountMinute(statODS.getOfficialAccountApplyAmount());
+            result.setOfficialAccountApplyRoiMinute(statODS.getOfficialAccountApplyRoi());
+            result.setOfficialAccountOrderCountMinute(statODS.getOfficialAccountOrderCount());
+            result.setOfficialAccountFirstDayOrderCountMinute(statODS.getOfficialAccountFirstDayOrderCount());
+            result.setOfficialAccountOrderUserCountMinute(statODS.getOfficialAccountOrderUserCount());
+            result.setOfficialAccountOrderRateMinute(statODS.getOfficialAccountOrderRate());
+            result.setOfficialAccountOrderCostMinute(statODS.getOfficialAccountOrderCost());
+            result.setOfficialAccountOrderAmountMinute(statODS.getOfficialAccountOrderAmount());
+            result.setOfficialAccountFirstDayOrderAmountMinute(statODS.getOfficialAccountFirstDayOrderAmount());
+            result.setOfficialAccountOrderRoiMinute(statODS.getOfficialAccountOrderRoi());
+            result.setOfficialAccountConsultCountMinute(statODS.getOfficialAccountConsultCount());
+            result.setOfficialAccountReaderCountMinute(statODS.getOfficialAccountReaderCount());
+            result.setOfficialAccountCreditApplyUserCountMinute(statODS.getOfficialAccountCreditApplyUserCount());
+            result.setOfficialAccountCreditUserCountMinute(statODS.getOfficialAccountCreditUserCount());
+            result.setForwardCountMinute(statODS.getForwardCount());
+            result.setForwardUserCountMinute(statODS.getForwardUserCount());
+            result.setNoInterestCountMinute(statODS.getNoInterestCount());
         }
         result.setAdGroupMapStr(PlanUtil.adGroupMapStr(adGroupMap));
         result.setAdIdsStr(PlanUtil.adIdsStr(statODS.getAdId()));

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/AdHourDTOStreamProcess.java

@@ -0,0 +1,61 @@
+package flink.zanxiangnet.ad.monitoring.process;
+
+import com.tencent.ads.model.HourlyReportsGetListStruct;
+import flink.zanxiangnet.ad.monitoring.pojo.dto.AdDataOfMinuteDTO;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfHourODS;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfMinuteODS;
+import flink.zanxiangnet.ad.monitoring.util.DateUtil;
+import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
+import org.springframework.beans.BeanUtils;
+
+import java.util.Date;
+
+/**
+ * 对小时数据分流
+ */
+public class AdHourDTOStreamProcess extends ProcessFunction<String, AdDataOfMinuteODS> {
+    final OutputTag<AdDataOfMinuteODS> adMinuteStreamTag;
+    final OutputTag<AdDataOfHourODS> adHourStreamTag;
+
+    public AdHourDTOStreamProcess(OutputTag<AdDataOfMinuteODS> adMinuteStreamTag, OutputTag<AdDataOfHourODS> adHourStreamTag) {
+        this.adMinuteStreamTag = adMinuteStreamTag;
+        this.adHourStreamTag = adHourStreamTag;
+    }
+
+    @Override
+    public void processElement(String adDataStr, ProcessFunction<String, AdDataOfMinuteODS>.Context context, Collector<AdDataOfMinuteODS> collector) throws Exception {
+        AdDataOfMinuteDTO dto = JsonUtil.toObj(adDataStr, AdDataOfMinuteDTO.class);
+        // 指记录被创建的时间
+        long createTime = dto.getCreateTime();
+        // 指记录统计的时间（如果是实时统计的数据精确到分钟，回滚的历史数据精确到天）
+        long statTime = DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(dto.getDataTime()).withSecond(0).withNano(0));
+        HourlyReportsGetListStruct struct = dto.getHourlyReportsGetListStruct();
+
+        if (createTime - statTime > (60 * 60 * 1000L)) {
+            AdDataOfHourODS adODS = new AdDataOfHourODS();
+            BeanUtils.copyProperties(struct, adODS);
+            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
+            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
+            adODS.setStatTime(DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(statTime).withMinute(0)));
+            adODS.setAccountId(dto.getAccountId());
+            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
+            adODS.setCreateTime(new Date(createTime));
+            adODS.removeNull();
+            context.output(adHourStreamTag, adODS);
+        } else {
+            AdDataOfMinuteODS adODS = new AdDataOfMinuteODS();
+            BeanUtils.copyProperties(struct, adODS);
+            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
+            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
+            adODS.setStatTime(statTime);
+            adODS.setAccountId(dto.getAccountId());
+            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
+            adODS.setCreateTime(new Date(createTime));
+            adODS.removeNull();
+            context.output(adMinuteStreamTag, adODS);
+        }
+    }
+}

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/AdHourDWDProcess.java

@@ -0,0 +1,96 @@
+package flink.zanxiangnet.ad.monitoring.process;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+import flink.zanxiangnet.ad.monitoring.maxcompute.MaxComputeLog;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfHourODS;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdStatOfHourDWD;
+import flink.zanxiangnet.ad.monitoring.pojo.properties.ApplicationProperties;
+import flink.zanxiangnet.ad.monitoring.util.DateUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
+import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
+import org.apache.flink.util.Collector;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class AdHourDWDProcess extends ProcessWindowFunction<AdDataOfHourODS, AdStatOfHourDWD, Long, GlobalWindow> {
+    private Odps odps;
+    // 上次查询的天数据
+    private ValueState<String> lastQueryDayState;
+    // 聚合的天的数据
+    private MapState<String, AdStatOfHourDWD> historyReduceState;
+
+    @Override
+    public void open(Configuration conf) {
+        Map<String, String> params = getRuntimeContext()
+                .getExecutionConfig()
+                .getGlobalJobParameters()
+                .toMap();
+        Account account = new AliyunAccount(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ID),
+                params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_KEY));
+        odps = new Odps(account);
+        odps.getRestClient().setRetryLogger(new MaxComputeLog());
+        odps.setEndpoint(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ENDPOINT));
+        odps.setDefaultProject(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_PROJECT_NAME));
+
+        lastQueryDayState = getRuntimeContext().getState(new ValueStateDescriptor<>("lastQueryDayState", String.class));
+        historyReduceState = getRuntimeContext().getMapState(new MapStateDescriptor<>("historyReduceState", Types.STRING, Types.POJO(AdStatOfHourDWD.class)));
+    }
+
+    @Override
+    public void process(Long elementsCount, ProcessWindowFunction<AdDataOfHourODS, AdStatOfHourDWD, Long, GlobalWindow>.Context context,
+                        Iterable<AdDataOfHourODS> iterable, Collector<AdStatOfHourDWD> collector) throws Exception {
+        AdDataOfHourODS element = iterable.iterator().next();
+        LocalDate statDay = DateUtil.parseLocalDate(element.getStatDay());
+        LocalDateTime statTime = LocalDateTime.of(statDay, LocalTime.of(element.getHour(), 0, 0));
+        long now = context.currentProcessingTime();
+        LocalDate today = LocalDate.now();
+
+        String lastQueryDay = lastQueryDayState.value();
+        // 从 maxCompute拉取指定 广告的历史数据
+        if (lastQueryDay == null || !lastQueryDay.equals(DateUtil.formatLocalDate(today))) {
+            LocalDate endDay = today, beginDay = statDay.minusDays(60);
+            String sql = "SELECT * FROM ad_stat_of_hour_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endDay) + "\" AND ad_id = " + element.getAdId() + ";";
+            Instance instance = SQLTask.run(odps, sql);
+            // log.error("sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+            instance.waitForSuccess();
+            List<Record> records = SQLTask.getResult(instance);
+            Map<String, AdStatOfHourDWD> historyHourMap = records.stream()
+                    .map(AdStatOfHourDWD::byMaxCompute)
+                    .sorted((o1, o2) -> new Long(o1.getCreateTime().getTime() - o2.getCreateTime().getTime()).intValue())
+                    .collect(Collectors.toMap(data -> data.getStatDay() + data.getHour(), data -> data, (val1, val2) -> val2));
+            historyReduceState.clear();
+            historyReduceState.putAll(historyHourMap);
+            lastQueryDayState.update(DateUtil.formatLocalDate(today));
+        }
+        AdStatOfHourDWD lastReduceData = null;
+        for (int i = 1; i < 60 * 24; i++) {
+            LocalDateTime time = statTime.minusHours(i);
+            lastReduceData = historyReduceState.get(DateUtil.formatLocalDate(time.toLocalDate()) + time.getHour());
+            if (lastReduceData != null) {
+                break;
+            }
+        }
+
+        AdStatOfHourDWD newStatData = AdStatOfHourDWD.reduce(lastReduceData, element, now);
+        historyReduceState.put(newStatData.getStatDay(), newStatData);
+        collector.collect(newStatData);
+    }
+}

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/AdMinuteDWDProcess.java

@@ -7,12 +7,11 @@ import com.aliyun.odps.account.AliyunAccount;
 import com.aliyun.odps.data.Record;
 import com.aliyun.odps.task.SQLTask;
 import flink.zanxiangnet.ad.monitoring.maxcompute.MaxComputeLog;
-import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfMinuteODS;
-import flink.zanxiangnet.ad.monitoring.pojo.entity.AdStatOfDayDWD;
-import flink.zanxiangnet.ad.monitoring.pojo.entity.AdStatOfMinuteDWD;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.*;
 import flink.zanxiangnet.ad.monitoring.pojo.properties.ApplicationProperties;
 import flink.zanxiangnet.ad.monitoring.util.DateUtil;
 import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.api.common.state.MapState;
 import org.apache.flink.api.common.state.MapStateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
@@ -21,6 +20,7 @@ import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -31,10 +31,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+@Slf4j
 public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS, AdStatOfMinuteDWD, Long, TimeWindow> {
     private static final DateTimeFormatter formatForLastReduceKey = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH:mm");
-    private Odps odps;
 
+    private final OutputTag<AdStatOfHourDWD> adHourFromMinuteStreamTag;
+
+    private Odps odps;
     // 历史的天数据
     private ValueState<AdStatOfDayDWD> historyDayState;
     // 上次查询的天数据
@@ -44,6 +47,10 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
     // 前 5分钟聚合的数据
     private MapState<String, AdStatOfMinuteDWD> lastReduceState;
 
+    public AdMinuteDWDProcess(OutputTag<AdStatOfHourDWD> adHourFromMinuteStreamTag) {
+        this.adHourFromMinuteStreamTag = adHourFromMinuteStreamTag;
+    }
+
     @Override
     public void open(Configuration conf) {
         Map<String, String> params = getRuntimeContext()
@@ -71,7 +78,7 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
         LocalDate beginDate = beginDateTime.toLocalDate();
         String statDay = DateUtil.formatLocalDate(beginDate);
         int hour = beginDateTime.getHour();
-        long now = System.currentTimeMillis();
+        long now = context.currentProcessingTime();
         AdDataOfMinuteODS statODS = null;
         List<AdDataOfMinuteODS> adDataOfMinuteODSList = new ArrayList<>(24);
         for (AdDataOfMinuteODS adDataOfMinuteODS : iterable) {
@@ -80,7 +87,6 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
                 continue;
             }
             statODS = adDataOfMinuteODS;
-            break;
         }
         if (statODS == null) {
             return;
@@ -97,7 +103,7 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
             LocalDate endDay = beginDate.minusDays(2L), beginDay = endDay.minusDays(30L);
             String sql = "SELECT * FROM ad_stat_of_day_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + endDay + "\" AND ad_id = " + adId + ";";
             Instance instance = SQLTask.run(odps, sql);
-            System.out.println("101===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+            // log.error("sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
             instance.waitForSuccess();
             List<Record> records = SQLTask.getResult(instance);
             List<AdStatOfDayDWD> historyDayData = records.stream().map(AdStatOfDayDWD::byMaxCompute).sorted((val1, val2) -> {
@@ -117,6 +123,9 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
         // 聚合当天的全部数据
         AdStatOfMinuteDWD newAdStat = AdStatOfMinuteDWD.reduce(beforeYesterdayDayDWD, yesterdayMinuteDWD, adDataOfMinuteODSList, statODS, lastReduceData, now);
         collector.collect(newAdStat);
+        if (beginDateTime.getMinute() == 55) {
+            context.output(adHourFromMinuteStreamTag, AdStatOfHourDWD.byMinuteDWD(newAdStat));
+        }
         lastReduceState.put(beginDateTime.format(formatForLastReduceKey), newAdStat);
         yesterdayMinuteState.put(statDay, newAdStat);
 
@@ -156,6 +165,5 @@ public class AdMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS,
 
     @Override
     public void clear(ProcessWindowFunction<AdDataOfMinuteODS, AdStatOfMinuteDWD, Long, TimeWindow>.Context context) throws Exception {
-        System.out.println("窗口关闭");
     }
 }

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/CostHourDayProcess.java

@@ -259,7 +259,7 @@ public class CostHourDayProcess extends ProcessWindowFunction<AdStatOfDayDWD, Co
                     "and dt='" + adStatOfMinuteDWD.getStatDay() + "' and hour='" + tmpHour + "'";
 
 
-//            System.out.println(sql);
+//            log.error(sql);
             Statement statement = connection.createStatement();
             ResultSet rs = statement.executeQuery(sql);
             while (rs.next()) {
@@ -296,16 +296,15 @@ public class CostHourDayProcess extends ProcessWindowFunction<AdStatOfDayDWD, Co
             CostHourDM costHourDM_new = datachange(adStatOfMinuteDWD, costHourDM);
 
             collector.collect(costHourDM_new);
-//            System.out.println("costhour_输出：" + JsonUtil.toString(costHourDM_new));
+//            log.error("costhour_输出：" + JsonUtil.toString(costHourDM_new));
         }
-//        System.out.println("costhour_windowCount：" + adStatOfMinuteDWDlist.size());
+//        log.error("costhour_windowCount：" + adStatOfMinuteDWDlist.size());
 
 
     }
 
     @Override
     public void clear(ProcessWindowFunction<AdStatOfDayDWD, CostHourDM, Long, GlobalWindow>.Context context) throws Exception {
-        System.out.println("窗口关闭");
     }
 
 }

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/PlanHourDTOStreamProcess.java

@@ -0,0 +1,63 @@
+package flink.zanxiangnet.ad.monitoring.process;
+
+import com.tencent.ads.model.HourlyReportsGetListStruct;
+import flink.zanxiangnet.ad.monitoring.pojo.dto.AdDataOfMinuteDTO;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfHourODS;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfMinuteODS;
+import flink.zanxiangnet.ad.monitoring.util.DateUtil;
+import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
+import org.springframework.beans.BeanUtils;
+
+import java.util.Date;
+
+@Slf4j
+public class PlanHourDTOStreamProcess extends ProcessFunction<String, AdDataOfMinuteODS> {
+
+    // 广告分钟数据（前 5分钟的广告消耗数据）
+    private final OutputTag<AdDataOfMinuteODS> adMinuteStreamTag;
+    // 广告小时数据（往前回滚 10天）
+    private final OutputTag<AdDataOfHourODS> adHourStreamTag;
+
+    public PlanHourDTOStreamProcess(OutputTag<AdDataOfMinuteODS> adMinuteStreamTag, OutputTag<AdDataOfHourODS> adHourStreamTag) {
+        this.adMinuteStreamTag = adMinuteStreamTag;
+        this.adHourStreamTag = adHourStreamTag;
+    }
+
+    @Override
+    public void processElement(String adDataStr, ProcessFunction<String, AdDataOfMinuteODS>.Context context, Collector<AdDataOfMinuteODS> collector) throws Exception {
+        AdDataOfMinuteDTO dto = JsonUtil.toObj(adDataStr, AdDataOfMinuteDTO.class);
+        // 指记录被创建的时间
+        long createTime = dto.getCreateTime();
+        // 指记录统计的时间（如果是实时统计的数据精确到分钟，回滚的历史数据精确到天）
+        long statTime = DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(dto.getDataTime()).withSecond(0).withNano(0));
+        HourlyReportsGetListStruct struct = dto.getHourlyReportsGetListStruct();
+
+        if (createTime - statTime > (60 * 60 * 1000L)) {
+            AdDataOfHourODS adODS = new AdDataOfHourODS();
+            BeanUtils.copyProperties(struct, adODS);
+            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
+            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
+            adODS.setStatTime(DateUtil.localDateTimeToMilli(DateUtil.milliToLocalDateTime(statTime).withMinute(0)));
+            adODS.setAccountId(dto.getAccountId());
+            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
+            adODS.setCreateTime(new Date(createTime));
+            adODS.removeNull();
+            context.output(adHourStreamTag, adODS);
+        } else {
+            AdDataOfMinuteODS adODS = new AdDataOfMinuteODS();
+            BeanUtils.copyProperties(struct, adODS);
+            adODS.setStatDay(DateUtil.formatLocalDate(DateUtil.milliToLocalDate(statTime)));
+            adODS.setHour(dto.getHourlyReportsGetListStruct().getHour().intValue());
+            adODS.setStatTime(statTime);
+            adODS.setAccountId(dto.getAccountId());
+            adODS.setAgencyAccountId(dto.getHourlyReportsGetListStruct().getAccountId());
+            adODS.setCreateTime(new Date(createTime));
+            adODS.removeNull();
+            context.output(adMinuteStreamTag, adODS);
+        }
+    }
+}

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/PlanHourDWDProcess.java

@@ -0,0 +1,97 @@
+package flink.zanxiangnet.ad.monitoring.process;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+import flink.zanxiangnet.ad.monitoring.maxcompute.MaxComputeLog;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.AdDataOfHourODS;
+import flink.zanxiangnet.ad.monitoring.pojo.entity.PlanStatOfHourDWD;
+import flink.zanxiangnet.ad.monitoring.pojo.properties.ApplicationProperties;
+import flink.zanxiangnet.ad.monitoring.util.DateUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
+import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
+import org.apache.flink.util.Collector;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class PlanHourDWDProcess extends ProcessWindowFunction<AdDataOfHourODS, PlanStatOfHourDWD, Long, GlobalWindow> {
+    
+    private Odps odps;
+    // 上次查询的天数据
+    private ValueState<String> lastQueryDayState;
+    // 聚合的天的数据
+    private MapState<String, PlanStatOfHourDWD> historyReduceState;
+
+    @Override
+    public void open(Configuration conf) {
+        Map<String, String> params = getRuntimeContext()
+                .getExecutionConfig()
+                .getGlobalJobParameters()
+                .toMap();
+        Account account = new AliyunAccount(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ID),
+                params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_KEY));
+        odps = new Odps(account);
+        odps.getRestClient().setRetryLogger(new MaxComputeLog());
+        odps.setEndpoint(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_ENDPOINT));
+        odps.setDefaultProject(params.get(ApplicationProperties.MAX_COMPUTE_ACCOUNT_PROJECT_NAME));
+
+        lastQueryDayState = getRuntimeContext().getState(new ValueStateDescriptor<>("lastQueryDayState", String.class));
+        historyReduceState = getRuntimeContext().getMapState(new MapStateDescriptor<>("historyReduceState", Types.STRING, Types.POJO(PlanStatOfHourDWD.class)));
+    }
+
+    @Override
+    public void process(Long elementsCount, ProcessWindowFunction<AdDataOfHourODS, PlanStatOfHourDWD, Long, GlobalWindow>.Context context,
+                        Iterable<AdDataOfHourODS> iterable, Collector<PlanStatOfHourDWD> collector) throws Exception {
+        AdDataOfHourODS element = iterable.iterator().next();
+        LocalDate statDay = DateUtil.parseLocalDate(element.getStatDay());
+        LocalDateTime statTime = LocalDateTime.of(statDay, LocalTime.of(element.getHour(), 0, 0));
+        long now = System.currentTimeMillis();
+        LocalDate today = LocalDate.now();
+
+        String lastQueryDay = lastQueryDayState.value();
+        // 从 maxCompute拉取指定 广告的历史数据
+        if (lastQueryDay == null || !lastQueryDay.equals(DateUtil.formatLocalDate(today))) {
+            LocalDate endDay = today, beginDay = statDay.minusDays(60);
+            String sql = "SELECT * FROM plan_stat_of_hour_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + DateUtil.formatLocalDate(endDay) + "\" AND campaign_id = " + element.getCampaignId() + ";";
+            Instance instance = SQLTask.run(odps, sql);
+            System.out.println("178===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+            instance.waitForSuccess();
+            List<Record> records = SQLTask.getResult(instance);
+            Map<String, PlanStatOfHourDWD> historyHourMap = records.stream()
+                    .map(PlanStatOfHourDWD::byMaxCompute)
+                    .sorted((o1, o2) -> new Long(o1.getCreateTime().getTime() - o2.getCreateTime().getTime()).intValue())
+                    .collect(Collectors.toMap(data -> data.getStatDay() + data.getHour(), data -> data, (val1, val2) -> val2));
+            historyReduceState.clear();
+            historyReduceState.putAll(historyHourMap);
+            lastQueryDayState.update(DateUtil.formatLocalDate(today));
+        }
+        PlanStatOfHourDWD lastReduceData = null;
+        for (int i = 1; i < 60 * 24; i++) {
+            LocalDateTime time = statTime.minusHours(i);
+            lastReduceData = historyReduceState.get(DateUtil.formatLocalDate(time.toLocalDate()) + time.getHour());
+            if (lastReduceData != null) {
+                break;
+            }
+        }
+
+        PlanStatOfHourDWD newStatData = PlanStatOfHourDWD.reduce(lastReduceData, element, now);
+        historyReduceState.put(newStatData.getStatDay(), newStatData);
+        collector.collect(newStatData);
+    }
+}

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/process/PlanMinuteDWDProcess.java

@@ -10,6 +10,7 @@ import flink.zanxiangnet.ad.monitoring.maxcompute.MaxComputeLog;
 import flink.zanxiangnet.ad.monitoring.pojo.entity.*;
 import flink.zanxiangnet.ad.monitoring.pojo.properties.ApplicationProperties;
 import flink.zanxiangnet.ad.monitoring.util.DateUtil;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.api.common.state.MapState;
 import org.apache.flink.api.common.state.MapStateDescriptor;
 import org.apache.flink.api.common.state.ValueState;
@@ -18,6 +19,7 @@ import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -28,8 +30,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+@Slf4j
 public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteODS, PlanStatOfMinuteDWD, Long, TimeWindow> {
     private static final DateTimeFormatter formatForLastReduceKey = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH:mm");
+
+    private final OutputTag<PlanStatOfHourDWD> planHourFromMinuteStreamTag;
+
     private Odps odps;
 
     // 历史的天数据
@@ -41,6 +47,10 @@ public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteOD
     // 前 5分钟聚合的数据
     private MapState<String, PlanStatOfMinuteDWD> lastReduceState;
 
+    public PlanMinuteDWDProcess(OutputTag<PlanStatOfHourDWD> planHourFromMinuteStreamTag) {
+        this.planHourFromMinuteStreamTag = planHourFromMinuteStreamTag;
+    }
+
     @Override
     public void open(Configuration conf) {
         Map<String, String> params = getRuntimeContext()
@@ -68,7 +78,7 @@ public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteOD
         LocalDate beginDate = beginDateTime.toLocalDate();
         String statDay = DateUtil.formatLocalDate(beginDate);
         int hour = beginDateTime.getHour();
-        long now = System.currentTimeMillis();
+        long now = context.currentProcessingTime();
         AdDataOfMinuteODS statODS = null;
         List<AdDataOfMinuteODS> adDataOfMinuteODSList = new ArrayList<>(24);
         for (AdDataOfMinuteODS adDataOfMinuteODS : iterable) {
@@ -77,7 +87,6 @@ public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteOD
                 continue;
             }
             statODS = adDataOfMinuteODS;
-            break;
         }
         if (statODS == null) {
             return;
@@ -94,7 +103,7 @@ public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteOD
             LocalDate endDay = beginDate.minusDays(2L), beginDay = endDay.minusDays(30L);
             String sql = "SELECT * FROM plan_stat_of_day_dwd WHERE stat_day >= \"" + DateUtil.formatLocalDate(beginDay) + "\" AND stat_day <= \"" + endDay + "\" AND campaign_id = " + campaignId + ";";
             Instance instance = SQLTask.run(odps, sql);
-            System.out.println("97===>sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
+            // log.error("sql: " + sql + ", odps日志: " + odps.logview().generateLogView(instance, 7 * 24));
             instance.waitForSuccess();
             List<Record> records = SQLTask.getResult(instance);
             List<PlanStatOfDayDWD> historyDayData = records.stream().map(PlanStatOfDayDWD::byMaxCompute).sorted((val1, val2) -> {
@@ -114,6 +123,9 @@ public class PlanMinuteDWDProcess extends ProcessWindowFunction<AdDataOfMinuteOD
         // 聚合当天的全部数据
         PlanStatOfMinuteDWD newAdStat = PlanStatOfMinuteDWD.reduce(beforeYesterdayDayDWD, yesterdayMinuteDWD, adDataOfMinuteODSList, statODS, lastReduceData, now);
         collector.collect(newAdStat);
+        if (beginDateTime.getMinute() == 55) {
+            context.output(planHourFromMinuteStreamTag, PlanStatOfHourDWD.byMinuteDWD(newAdStat));
+        }
         lastReduceState.put(beginDateTime.format(formatForLastReduceKey), newAdStat);
         yesterdayMinuteState.put(statDay, newAdStat);
 

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/trigger/AdMinuteODSStreamTrigger.java

@@ -7,14 +7,16 @@ import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 
 public class AdMinuteODSStreamTrigger extends Trigger<AdDataOfMinuteODS, TimeWindow> {
+    /**
+     * @param adDataOfMinuteODS
+     * @param time              此处指 eventTime
+     * @param timeWindow        start：窗口开始时间，end：窗口关闭时间，maxTimestamp：能接收到数据的的最大时间
+     * @param triggerContext    currentWatermark：当前水印时间，第一个元素进入的时候为 Long.MAX
+     * @return
+     * @throws Exception
+     */
     @Override
     public TriggerResult onElement(AdDataOfMinuteODS adDataOfMinuteODS, long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        // FIXME 水印时间没有意义！拿到数据是 Long.MAX
-        if (timeWindow.maxTimestamp() <= triggerContext.getCurrentWatermark()) {
-            // 到了窗口的最大生命周期
-            return TriggerResult.FIRE_AND_PURGE;
-        }
-        triggerContext.registerEventTimeTimer(timeWindow.maxTimestamp());
         if (adDataOfMinuteODS.getHour() == DateUtil.milliToLocalDateTime(time).getHour()) {
             return TriggerResult.FIRE;
         }
@@ -26,13 +28,19 @@ public class AdMinuteODSStreamTrigger extends Trigger<AdDataOfMinuteODS, TimeWin
         return TriggerResult.CONTINUE;
     }
 
+    /**
+     * @param time           eventTime
+     * @param timeWindow
+     * @param triggerContext
+     * @return
+     * @throws Exception
+     */
     @Override
     public TriggerResult onEventTime(long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        return time == timeWindow.maxTimestamp() ? TriggerResult.FIRE : TriggerResult.CONTINUE;
+        return TriggerResult.FIRE_AND_PURGE;
     }
 
     @Override
     public void clear(TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        triggerContext.deleteEventTimeTimer(timeWindow.maxTimestamp());
     }
 }

flink-ad-monitoring/src/main/java/flink/zanxiangnet/ad/monitoring/trigger/PlanMinuteODSStreamTrigger.java

@@ -9,12 +9,6 @@ import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 public class PlanMinuteODSStreamTrigger extends Trigger<AdDataOfMinuteODS, TimeWindow> {
     @Override
     public TriggerResult onElement(AdDataOfMinuteODS adDataOfMinuteODS, long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        // FIXME 水印时间没有意义！拿到数据是 Long.MAX
-        if (timeWindow.maxTimestamp() <= triggerContext.getCurrentWatermark()) {
-            // 到了窗口的最大生命周期
-            return TriggerResult.FIRE_AND_PURGE;
-        }
-        triggerContext.registerEventTimeTimer(timeWindow.maxTimestamp());
         if (adDataOfMinuteODS.getHour() == DateUtil.milliToLocalDateTime(time).getHour()) {
             return TriggerResult.FIRE;
         }
@@ -28,11 +22,10 @@ public class PlanMinuteODSStreamTrigger extends Trigger<AdDataOfMinuteODS, TimeW
 
     @Override
     public TriggerResult onEventTime(long time, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        return time == timeWindow.maxTimestamp() ? TriggerResult.FIRE : TriggerResult.CONTINUE;
+        return TriggerResult.FIRE_AND_PURGE;
     }
 
     @Override
     public void clear(TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
-        triggerContext.deleteEventTimeTimer(timeWindow.maxTimestamp());
     }
 }

flink-ad-monitoring/src/main/resources/log4j2.properties

@@ -25,4 +25,5 @@ logger.sink.level = INFO
 appender.console.name = ConsoleAppender
 appender.console.type = CONSOLE
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+# appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5level %logger{20}(%method:%line) - %msg%n