|
@@ -1,6 +1,8 @@
|
|
package flink.zanxiangnet.ad.monitoring;
|
|
package flink.zanxiangnet.ad.monitoring;
|
|
|
|
|
|
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
|
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
|
|
|
+import flink.zanxiangnet.ad.monitoring.stream.BatchStream;
|
|
|
|
+import flink.zanxiangnet.ad.monitoring.stream.KeyedBatchStream;
|
|
import flink.zanxiangnet.ad.monitoring.util.DateUtil;
|
|
import flink.zanxiangnet.ad.monitoring.util.DateUtil;
|
|
import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
|
|
import flink.zanxiangnet.ad.monitoring.util.JsonUtil;
|
|
import lombok.AllArgsConstructor;
|
|
import lombok.AllArgsConstructor;
|
|
@@ -9,28 +11,21 @@ import lombok.Data;
|
|
import lombok.NoArgsConstructor;
|
|
import lombok.NoArgsConstructor;
|
|
import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
|
|
import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
|
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
|
-import org.apache.flink.api.common.functions.AggregateFunction;
|
|
|
|
-import org.apache.flink.api.common.functions.ReduceFunction;
|
|
|
|
-import org.apache.flink.api.common.functions.RichAggregateFunction;
|
|
|
|
-import org.apache.flink.api.common.functions.RichReduceFunction;
|
|
|
|
|
|
+import org.apache.flink.api.java.functions.KeySelector;
|
|
|
|
+import org.apache.flink.api.java.tuple.Tuple2;
|
|
import org.apache.flink.api.java.tuple.Tuple3;
|
|
import org.apache.flink.api.java.tuple.Tuple3;
|
|
-import org.apache.flink.api.java.tuple.Tuple5;
|
|
|
|
import org.apache.flink.configuration.Configuration;
|
|
import org.apache.flink.configuration.Configuration;
|
|
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
|
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
|
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
|
|
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
|
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
|
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
|
|
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
|
|
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
|
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
|
-import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
|
|
|
|
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
|
|
|
|
-import org.apache.flink.streaming.api.windowing.time.Time;
|
|
|
|
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
|
|
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
|
|
import org.apache.flink.util.Collector;
|
|
import org.apache.flink.util.Collector;
|
|
|
|
|
|
import java.time.Duration;
|
|
import java.time.Duration;
|
|
import java.time.LocalDateTime;
|
|
import java.time.LocalDateTime;
|
|
import java.util.ArrayList;
|
|
import java.util.ArrayList;
|
|
-import java.util.HashMap;
|
|
|
|
import java.util.List;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
@@ -78,12 +73,13 @@ public class Test {
|
|
return new Tuple5<>(tuple5.f0, Math.min(acc.f1, tuple5.f1), Math.max(acc.f2, tuple5.f2), tuple5.f3 + acc.f3, temp);
|
|
return new Tuple5<>(tuple5.f0, Math.min(acc.f1, tuple5.f1), Math.max(acc.f2, tuple5.f2), tuple5.f3 + acc.f3, temp);
|
|
}
|
|
}
|
|
}).print();*/
|
|
}).print();*/
|
|
- pojoStream.keyBy(Pojo::getUserId).countWindow(1)
|
|
|
|
|
|
+ SingleOutputStreamOperator<String> ff = pojoStream.keyBy(Pojo::getUserId).countWindow(1)
|
|
.process(new ProcessWindowFunction<Pojo, String, Integer, GlobalWindow>() {
|
|
.process(new ProcessWindowFunction<Pojo, String, Integer, GlobalWindow>() {
|
|
private final Map<Integer, Tuple3<LocalDateTime, LocalDateTime, List<Pojo>>> oldData = new ConcurrentHashMap<>();
|
|
private final Map<Integer, Tuple3<LocalDateTime, LocalDateTime, List<Pojo>>> oldData = new ConcurrentHashMap<>();
|
|
|
|
|
|
@Override
|
|
@Override
|
|
- public void process(Integer integer, ProcessWindowFunction<Pojo, String, Integer, GlobalWindow>.Context context, Iterable<Pojo> elements, Collector<String> out) throws Exception {
|
|
|
|
|
|
+ public void process(Integer integer, ProcessWindowFunction<Pojo, String, Integer, GlobalWindow>.Context context,
|
|
|
|
+ Iterable<Pojo> elements, Collector<String> out) throws Exception {
|
|
Pojo pojo = elements.iterator().next();
|
|
Pojo pojo = elements.iterator().next();
|
|
LocalDateTime createTime = DateUtil.milliToLocalDateTime(pojo.getCreateTime());
|
|
LocalDateTime createTime = DateUtil.milliToLocalDateTime(pojo.getCreateTime());
|
|
Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> temp = oldData.get(pojo.getUserId());
|
|
Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> temp = oldData.get(pojo.getUserId());
|
|
@@ -95,7 +91,9 @@ public class Test {
|
|
oldData.put(pojo.getUserId(), temp);
|
|
oldData.put(pojo.getUserId(), temp);
|
|
out.collect(JsonUtil.toString(temp.f2.stream().map(Pojo::getIndex).collect(Collectors.toList())));
|
|
out.collect(JsonUtil.toString(temp.f2.stream().map(Pojo::getIndex).collect(Collectors.toList())));
|
|
}
|
|
}
|
|
- }).print();
|
|
|
|
|
|
+ });
|
|
|
|
+ new BatchStream<>("ff", ff, 10L, 60 * 1000L).toBatch().print();
|
|
|
|
+ // new BatchSink<>(ff, 60 * 1000L, 10L).toBatch().print();
|
|
/* .apply(new WindowFunction<Pojo, String, Integer, GlobalWindow>() {
|
|
/* .apply(new WindowFunction<Pojo, String, Integer, GlobalWindow>() {
|
|
private Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> oldData = null;
|
|
private Tuple3<LocalDateTime, LocalDateTime, List<Pojo>> oldData = null;
|
|
@Override
|
|
@Override
|