org.apache.flink.api.common.serialization.Encoder Java Examples
The following examples show how to use
org.apache.flink.api.common.serialization.Encoder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TestUtils.java From flink with Apache License 2.0 | 6 votes |
|
static <ID> OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomizedRescalingTestSink(
final File outDir,
final int totalParallelism,
final int taskIdx,
final long bucketCheckInterval,
final BucketAssigner<Tuple2<String, Integer>, ID> bucketer,
final Encoder<Tuple2<String, Integer>> writer,
final RollingPolicy<Tuple2<String, Integer>, ID> rollingPolicy,
final BucketFactory<Tuple2<String, Integer>, ID> bucketFactory) throws Exception {
StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink
.forRowFormat(new Path(outDir.toURI()), writer)
.withNewBucketAssignerAndPolicy(bucketer, rollingPolicy)
.withBucketCheckInterval(bucketCheckInterval)
.withBucketFactory(bucketFactory)
.build();
return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx);
}
Example #2
| Source File: TestUtils.java From flink with Apache License 2.0 | 6 votes |
|
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createRescalingTestSink(
final File outDir,
final int totalParallelism,
final int taskIdx,
final long bucketCheckInterval,
final BucketAssigner<Tuple2<String, Integer>, String> bucketer,
final Encoder<Tuple2<String, Integer>> writer,
final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy,
final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception {
StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink
.forRowFormat(new Path(outDir.toURI()), writer)
.withBucketAssigner(bucketer)
.withRollingPolicy(rollingPolicy)
.withBucketCheckInterval(bucketCheckInterval)
.withBucketFactory(bucketFactory)
.build();
return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx);
}
Example #3
| Source File: StreamingFileSink.java From flink with Apache License 2.0 | 6 votes |
|
protected RowFormatBuilder(
Path basePath,
Encoder<IN> encoder,
BucketAssigner<IN, BucketID> assigner,
RollingPolicy<IN, BucketID> policy,
long bucketCheckInterval,
BucketFactory<IN, BucketID> bucketFactory,
OutputFileConfig outputFileConfig) {
this.basePath = Preconditions.checkNotNull(basePath);
this.encoder = Preconditions.checkNotNull(encoder);
this.bucketAssigner = Preconditions.checkNotNull(assigner);
this.rollingPolicy = Preconditions.checkNotNull(policy);
this.bucketCheckInterval = bucketCheckInterval;
this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
this.outputFileConfig = Preconditions.checkNotNull(outputFileConfig);
}
Example #4
| Source File: TestUtils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomRescalingTestSink(
final File outDir,
final int totalParallelism,
final int taskIdx,
final long bucketCheckInterval,
final BucketAssigner<Tuple2<String, Integer>, String> bucketer,
final Encoder<Tuple2<String, Integer>> writer,
final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy,
final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception {
StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink
.forRowFormat(new Path(outDir.toURI()), writer)
.withBucketAssigner(bucketer)
.withRollingPolicy(rollingPolicy)
.withBucketCheckInterval(bucketCheckInterval)
.withBucketFactory(bucketFactory)
.build();
return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx);
}
Example #5
| Source File: TestUtils.java From flink with Apache License 2.0 | 6 votes |
|
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomRescalingTestSink(
final File outDir,
final int totalParallelism,
final int taskIdx,
final long bucketCheckInterval,
final BucketAssigner<Tuple2<String, Integer>, String> bucketer,
final Encoder<Tuple2<String, Integer>> writer,
final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy,
final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception {
StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink
.forRowFormat(new Path(outDir.toURI()), writer)
.withBucketAssigner(bucketer)
.withRollingPolicy(rollingPolicy)
.withBucketCheckInterval(bucketCheckInterval)
.withBucketFactory(bucketFactory)
.build();
return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx);
}
Example #6
| Source File: StreamingFileSink.java From flink with Apache License 2.0 | 6 votes |
|
private RowFormatBuilder(
Path basePath,
Encoder<IN> encoder,
BucketAssigner<IN, BucketID> assigner,
RollingPolicy<IN, BucketID> policy,
long bucketCheckInterval,
BucketFactory<IN, BucketID> bucketFactory,
String partFilePrefix,
String partFileSuffix) {
this.basePath = Preconditions.checkNotNull(basePath);
this.encoder = Preconditions.checkNotNull(encoder);
this.bucketAssigner = Preconditions.checkNotNull(assigner);
this.rollingPolicy = Preconditions.checkNotNull(policy);
this.bucketCheckInterval = bucketCheckInterval;
this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
this.partFilePrefix = Preconditions.checkNotNull(partFilePrefix);
this.partFileSuffix = Preconditions.checkNotNull(partFileSuffix);
}
Example #7
| Source File: FileSystemTableSink.java From flink with Apache License 2.0 | 5 votes |
|
private static OutputFormat<RowData> createEncoderOutputFormat(
Encoder<RowData> encoder,
Path path) {
return new OutputFormat<RowData>() {
private static final long serialVersionUID = 1L;
private transient FSDataOutputStream output;
@Override
public void configure(Configuration parameters) {
}
@Override
public void open(int taskNumber, int numTasks) throws IOException {
this.output = path.getFileSystem()
.create(path, FileSystem.WriteMode.OVERWRITE);
}
@Override
public void writeRecord(RowData record) throws IOException {
encoder.encode(record, output);
}
@Override
public void close() throws IOException {
this.output.flush();
this.output.close();
}
};
}
Example #8
| Source File: RowWisePartWriter.java From flink with Apache License 2.0 | 5 votes |
|
RowWisePartWriter(
final BucketID bucketId,
final RecoverableFsDataOutputStream currentPartStream,
final Encoder<IN> encoder,
final long creationTime) {
super(bucketId, currentPartStream, creationTime);
this.encoder = Preconditions.checkNotNull(encoder);
}
Example #9
| Source File: CsvFileSystemFormatFactory.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
ReadableConfig options = context.getFormatOptions();
validateFormatOptions(options);
CsvRowDataSerializationSchema.Builder builder = new CsvRowDataSerializationSchema.Builder(
context.getFormatRowType());
options.getOptional(FIELD_DELIMITER).map(s -> s.charAt(0))
.ifPresent(builder::setFieldDelimiter);
options.getOptional(LINE_DELIMITER)
.ifPresent(builder::setLineDelimiter);
if (options.get(DISABLE_QUOTE_CHARACTER)) {
builder.disableQuoteCharacter();
} else {
options.getOptional(QUOTE_CHARACTER).map(s -> s.charAt(0)).ifPresent(builder::setQuoteCharacter);
}
options.getOptional(ARRAY_ELEMENT_DELIMITER)
.ifPresent(builder::setArrayElementDelimiter);
options.getOptional(ESCAPE_CHARACTER).map(s -> s.charAt(0))
.ifPresent(builder::setEscapeCharacter);
options.getOptional(NULL_LITERAL)
.ifPresent(builder::setNullLiteral);
final CsvRowDataSerializationSchema serializationSchema = builder.build();
return Optional.of((record, stream) -> stream.write(serializationSchema.serialize(record)));
}
Example #10
| Source File: TestCsvFileSystemFormatFactory.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
if (useBulkWriter(context)) {
return Optional.empty();
}
DataType[] types = context.getFormatFieldTypes();
return Optional.of((rowData, stream) -> {
writeCsvToStream(types, rowData, stream);
});
}
Example #11
| Source File: FileSystemTableSink.java From flink with Apache License 2.0 | 5 votes |
|
private Object createWriter() {
FileSystemFormatFactory formatFactory = createFormatFactory(properties);
Configuration conf = new Configuration();
properties.forEach(conf::setString);
FileSystemFormatFactory.WriterContext context = new FileSystemFormatFactory.WriterContext() {
@Override
public TableSchema getSchema() {
return schema;
}
@Override
public ReadableConfig getFormatOptions() {
return new DelegatingConfiguration(conf, formatFactory.factoryIdentifier() + ".");
}
@Override
public List<String> getPartitionKeys() {
return partitionKeys;
}
};
Optional<Encoder<RowData>> encoder = formatFactory.createEncoder(context);
Optional<BulkWriter.Factory<RowData>> bulk = formatFactory.createBulkWriterFactory(context);
if (encoder.isPresent()) {
return encoder.get();
} else if (bulk.isPresent()) {
return bulk.get();
} else {
throw new TableException(
formatFactory + " format should implement at least one Encoder or BulkWriter");
}
}
Example #12
| Source File: FileSystemTableSink.java From flink with Apache License 2.0 | 5 votes |
|
@SuppressWarnings("unchecked")
private OutputFormatFactory<RowData> createOutputFormatFactory() {
Object writer = createWriter();
return writer instanceof Encoder ?
path -> createEncoderOutputFormat((Encoder<RowData>) writer, path) :
path -> createBulkWriterOutputFormat((BulkWriter.Factory<RowData>) writer, path);
}
Example #13
| Source File: TestUtils.java From flink with Apache License 2.0 | 5 votes |
|
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createRescalingTestSink(
File outDir,
int totalParallelism,
int taskIdx,
long inactivityInterval,
long partMaxSize) throws Exception {
final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy =
DefaultRollingPolicy
.create()
.withMaxPartSize(partMaxSize)
.withRolloverInterval(inactivityInterval)
.withInactivityInterval(inactivityInterval)
.build();
final BucketAssigner<Tuple2<String, Integer>, String> bucketer = new TupleToStringBucketer();
final Encoder<Tuple2<String, Integer>> encoder = (element, stream) -> {
stream.write((element.f0 + '@' + element.f1).getBytes(StandardCharsets.UTF_8));
stream.write('\n');
};
return createCustomRescalingTestSink(
outDir,
totalParallelism,
taskIdx,
10L,
bucketer,
encoder,
rollingPolicy,
new DefaultBucketFactoryImpl<>());
}
Example #14
| Source File: RowWisePartWriter.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private RowWisePartWriter(
final BucketID bucketId,
final RecoverableFsDataOutputStream currentPartStream,
final Encoder<IN> encoder,
final long creationTime) {
super(bucketId, currentPartStream, creationTime);
this.encoder = Preconditions.checkNotNull(encoder);
}
Example #15
| Source File: RowWisePartWriter.java From flink with Apache License 2.0 | 5 votes |
|
private RowWisePartWriter(
final BucketID bucketId,
final RecoverableFsDataOutputStream currentPartStream,
final Encoder<IN> encoder,
final long creationTime) {
super(bucketId, currentPartStream, creationTime);
this.encoder = Preconditions.checkNotNull(encoder);
}
Example #16
| Source File: TestUtils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createRescalingTestSink(
File outDir,
int totalParallelism,
int taskIdx,
long inactivityInterval,
long partMaxSize) throws Exception {
final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy =
DefaultRollingPolicy
.create()
.withMaxPartSize(partMaxSize)
.withRolloverInterval(inactivityInterval)
.withInactivityInterval(inactivityInterval)
.build();
final BucketAssigner<Tuple2<String, Integer>, String> bucketer = new TupleToStringBucketer();
final Encoder<Tuple2<String, Integer>> encoder = (element, stream) -> {
stream.write((element.f0 + '@' + element.f1).getBytes(StandardCharsets.UTF_8));
stream.write('\n');
};
return createCustomRescalingTestSink(
outDir,
totalParallelism,
taskIdx,
10L,
bucketer,
encoder,
rollingPolicy,
new DefaultBucketFactoryImpl<>());
}
Example #17
| Source File: StreamingFileSink.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private RowFormatBuilder(
Path basePath,
Encoder<IN> encoder,
BucketAssigner<IN, BucketID> assigner,
RollingPolicy<IN, BucketID> policy,
long bucketCheckInterval,
BucketFactory<IN, BucketID> bucketFactory) {
this.basePath = Preconditions.checkNotNull(basePath);
this.encoder = Preconditions.checkNotNull(encoder);
this.bucketAssigner = Preconditions.checkNotNull(assigner);
this.rollingPolicy = Preconditions.checkNotNull(policy);
this.bucketCheckInterval = bucketCheckInterval;
this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
}
Example #18
| Source File: JsonFileSystemFormatFactory.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
return Optional.of(new JsonRowDataEncoder(new JsonRowDataSerializationSchema(context.getFormatRowType(),
JsonOptions.getTimestampFormat(context.getFormatOptions()))));
}
Example #19
| Source File: RowWisePartWriter.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
Factory(Encoder<IN> encoder) {
this.encoder = encoder;
}
Example #20
| Source File: StreamingFileSink.java From flink with Apache License 2.0 | 4 votes |
|
private DefaultRowFormatBuilder(Path basePath, Encoder<IN> encoder, BucketAssigner<IN, String> bucketAssigner) {
super(basePath, encoder, bucketAssigner);
}
Example #21
| Source File: StreamingFileSink.java From flink with Apache License 2.0 | 4 votes |
|
protected RowFormatBuilder(Path basePath, Encoder<IN> encoder, BucketAssigner<IN, BucketID> bucketAssigner) {
this(basePath, encoder, bucketAssigner, DefaultRollingPolicy.builder().build(), DEFAULT_BUCKET_CHECK_INTERVAL, new DefaultBucketFactoryImpl<>(), OutputFileConfig.builder().build());
}
Example #22
| Source File: RowWiseBucketWriter.java From flink with Apache License 2.0 | 4 votes |
|
RowWiseBucketWriter(final RecoverableWriter recoverableWriter, final Encoder<IN> encoder) {
super(recoverableWriter);
this.encoder = encoder;
}
Example #23
| Source File: StreamingFileSink.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
RowFormatBuilder(Path basePath, Encoder<IN> encoder, BucketAssigner<IN, BucketID> bucketAssigner) {
this(basePath, encoder, bucketAssigner, DefaultRollingPolicy.create().build(), 60L * 1000L, new DefaultBucketFactoryImpl<>());
}
Example #24
| Source File: StreamSQLTestProgram.java From flink with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
ParameterTool params = ParameterTool.fromArgs(args);
String outputPath = params.getRequired("outputPath");
String planner = params.get("planner", "blink");
final EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance();
builder.inStreamingMode();
if (planner.equals("old")) {
builder.useOldPlanner();
} else if (planner.equals("blink")) {
builder.useBlinkPlanner();
}
final EnvironmentSettings settings = builder.build();
final StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(
3,
Time.of(10, TimeUnit.SECONDS)
));
sEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
sEnv.enableCheckpointing(4000);
sEnv.getConfig().setAutoWatermarkInterval(1000);
final StreamTableEnvironment tEnv = StreamTableEnvironment.create(sEnv, settings);
((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table1", new GeneratorTableSource(10, 100, 60, 0));
((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table2", new GeneratorTableSource(5, 0.2f, 60, 5));
int overWindowSizeSeconds = 1;
int tumbleWindowSizeSeconds = 10;
String overQuery = String.format(
"SELECT " +
" key, " +
" rowtime, " +
" COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " +
"FROM table1",
overWindowSizeSeconds);
String tumbleQuery = String.format(
"SELECT " +
" key, " +
" CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " +
" TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " +
" TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " +
"FROM (%s) " +
"WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " +
"GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)",
tumbleWindowSizeSeconds,
tumbleWindowSizeSeconds,
overQuery,
tumbleWindowSizeSeconds);
String joinQuery = String.format(
"SELECT " +
" t1.key, " +
" t2.rowtime AS rowtime, " +
" t2.correct," +
" t2.wStart " +
"FROM table2 t1, (%s) t2 " +
"WHERE " +
" t1.key = t2.key AND " +
" t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND",
tumbleQuery,
tumbleWindowSizeSeconds);
String finalAgg = String.format(
"SELECT " +
" SUM(correct) AS correct, " +
" TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " +
"FROM (%s) " +
"GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)",
joinQuery);
// get Table for SQL query
Table result = tEnv.sqlQuery(finalAgg);
// convert Table into append-only DataStream
DataStream<Row> resultStream =
tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP));
final StreamingFileSink<Row> sink = StreamingFileSink
.forRowFormat(new Path(outputPath), (Encoder<Row>) (element, stream) -> {
PrintStream out = new PrintStream(stream);
out.println(element.toString());
})
.withBucketAssigner(new KeyBucketAssigner())
.withRollingPolicy(OnCheckpointRollingPolicy.build())
.build();
resultStream
// inject a KillMapper that forwards all records but terminates the first execution attempt
.map(new KillMapper()).setParallelism(1)
// add sink function
.addSink(sink).setParallelism(1);
sEnv.execute();
}
Example #25
| Source File: OrcFileSystemFormatFactory.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
return Optional.empty();
}
Example #26
| Source File: ParquetFileSystemFormatFactory.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
return Optional.empty();
}
Example #27
| Source File: StreamingFileSink.java From flink with Apache License 2.0 | 4 votes |
|
RowFormatBuilder(Path basePath, Encoder<IN> encoder, BucketAssigner<IN, BucketID> bucketAssigner) {
this(basePath, encoder, bucketAssigner, DefaultRollingPolicy.create().build(), 60L * 1000L, new DefaultBucketFactoryImpl<>(), PartFileConfig.DEFAULT_PART_PREFIX, PartFileConfig.DEFAULT_PART_SUFFIX);
}
Example #28
| Source File: AvroFileSystemFormatFactory.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public Optional<Encoder<RowData>> createEncoder(WriterContext context) {
return Optional.empty();
}
Example #29
| Source File: FileSystemTableSink.java From flink with Apache License 2.0 | 4 votes |
|
private ProjectionEncoder(Encoder<RowData> encoder, RowDataPartitionComputer computer) {
this.encoder = encoder;
this.computer = computer;
}
Example #30
| Source File: StreamSQLTestProgram.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
ParameterTool params = ParameterTool.fromArgs(args);
String outputPath = params.getRequired("outputPath");
StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(
3,
Time.of(10, TimeUnit.SECONDS)
));
sEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
sEnv.enableCheckpointing(4000);
sEnv.getConfig().setAutoWatermarkInterval(1000);
StreamTableEnvironment tEnv = StreamTableEnvironment.create(sEnv);
tEnv.registerTableSource("table1", new GeneratorTableSource(10, 100, 60, 0));
tEnv.registerTableSource("table2", new GeneratorTableSource(5, 0.2f, 60, 5));
int overWindowSizeSeconds = 1;
int tumbleWindowSizeSeconds = 10;
String overQuery = String.format(
"SELECT " +
" key, " +
" rowtime, " +
" COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " +
"FROM table1",
overWindowSizeSeconds);
String tumbleQuery = String.format(
"SELECT " +
" key, " +
" CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " +
" TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " +
" TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " +
"FROM (%s) " +
"WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " +
"GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)",
tumbleWindowSizeSeconds,
tumbleWindowSizeSeconds,
overQuery,
tumbleWindowSizeSeconds);
String joinQuery = String.format(
"SELECT " +
" t1.key, " +
" t2.rowtime AS rowtime, " +
" t2.correct," +
" t2.wStart " +
"FROM table2 t1, (%s) t2 " +
"WHERE " +
" t1.key = t2.key AND " +
" t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND",
tumbleQuery,
tumbleWindowSizeSeconds);
String finalAgg = String.format(
"SELECT " +
" SUM(correct) AS correct, " +
" TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " +
"FROM (%s) " +
"GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)",
joinQuery);
// get Table for SQL query
Table result = tEnv.sqlQuery(finalAgg);
// convert Table into append-only DataStream
DataStream<Row> resultStream =
tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP));
final StreamingFileSink<Row> sink = StreamingFileSink
.forRowFormat(new Path(outputPath), (Encoder<Row>) (element, stream) -> {
PrintStream out = new PrintStream(stream);
out.println(element.toString());
})
.withBucketAssigner(new KeyBucketAssigner())
.withRollingPolicy(OnCheckpointRollingPolicy.build())
.build();
resultStream
// inject a KillMapper that forwards all records but terminates the first execution attempt
.map(new KillMapper()).setParallelism(1)
// add sink function
.addSink(sink).setParallelism(1);
sEnv.execute();
}
