Java Code Examples for org.apache.flink.streaming.api.functions.sink.SinkFunction

The following examples show how to use org.apache.flink.streaming.api.functions.sink.SinkFunction. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example 2
Source Project: flink   Source File: GroupedProcessingTimeWindowExample.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 
Example 3
@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	dataStream.addSink(sinkFunction)
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 
Example 4
Source Project: Flink-CEPplus   Source File: StreamCheckpointingITCase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Runs the following program.
 * <pre>
 *     [ (source)->(filter) ]-s->[ (map) ] -> [ (map) ] -> [ (groupBy/count)->(sink) ]
 * </pre>
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {
	DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS));

	stream
			// -------------- first vertex, chained to the source ----------------
			.filter(new StringRichFilterFunction())
			.shuffle()

			// -------------- seconds vertex - the stateful one that also fails ----------------
			.map(new StringPrefixCountRichMapFunction())
			.startNewChain()
			.map(new StatefulCounterFunction())

			// -------------- third vertex - counter and the sink ----------------
			.keyBy("prefix")
			.map(new OnceFailingPrefixCounter(NUM_STRINGS))
			.addSink(new SinkFunction<PrefixCount>() {

				@Override
				public void invoke(PrefixCount value) throws Exception {
					// Do nothing here
				}
			});
}
 
Example 5
Source Project: Flink-CEPplus   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example 6
public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 
Example 7
Source Project: flink   Source File: ElasticsearchUpsertTableSinkBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	return dataStream.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 
Example 8
Source Project: flink   Source File: StreamGraphGeneratorTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 
Example 9
Source Project: bahir-flink   Source File: ActiveMQConnectorITCase.java    License: Apache License 2.0 6 votes vote down vote up
private void createConsumerTopology(StreamExecutionEnvironment env, AMQSourceConfig<String> config) {
    AMQSource<String> source = new AMQSource<>(config);

    env.addSource(source)
        .addSink(new SinkFunction<String>() {
            final HashSet<Integer> set = new HashSet<>();
            @Override
            public void invoke(String value, Context context) throws Exception {
                int val = Integer.parseInt(value.split("-")[1]);
                set.add(val);

                if (set.size() == MESSAGES_NUM) {
                    throw new SuccessException();
                }
            }
        });
}
 
Example 10
Source Project: flink   Source File: DataStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example 11
Source Project: flink   Source File: Buckets.java    License: Apache License 2.0 6 votes vote down vote up
Bucket<IN, BucketID> onElement(final IN value, final SinkFunction.Context context) throws Exception {
	final long currentProcessingTime = context.currentProcessingTime();

	// setting the values in the bucketer context
	bucketerContext.update(
			context.timestamp(),
			context.currentWatermark(),
			currentProcessingTime);

	final BucketID bucketId = bucketAssigner.getBucketId(value, bucketerContext);
	final Bucket<IN, BucketID> bucket = getOrCreateBucketForBucketId(bucketId);
	bucket.write(value, currentProcessingTime);

	// we update the global max counter here because as buckets become inactive and
	// get removed from the list of active buckets, at the time when we want to create
	// another part file for the bucket, if we start from 0 we may overwrite previous parts.

	this.maxPartCounter = Math.max(maxPartCounter, bucket.getPartCounter());
	return bucket;
}
 
Example 12
Source Project: flink   Source File: StreamGraphGeneratorTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 
Example 13
Source Project: stateful-functions   Source File: EgressToSinkTranslator.java    License: Apache License 2.0 6 votes vote down vote up
private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 
Example 14
Source Project: stateful-functions   Source File: KafkaSinkProvider.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.put("bootstrap.servers", spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.put("transaction.timeout.ms", spec.transactionTimeoutDuration().toMillis());
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 
Example 15
Source Project: alchemy   Source File: Elasticsearch6TableSink.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
    Elasticsearch6SinkFunction upsertFunction = new Elasticsearch6SinkFunction(
            index,
            findIndex(indexField, schema.getFieldNames()),
            docType,
            keyDelimiter,
            keyNullLiteral,
            serializationSchema,
            contentType,
            requestFactory,
            keyFieldIndices);
    final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
            hosts,
            failureHandler,
            upsertFunction);
    dataStream.addSink(sinkFunction)
            .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));

}
 
Example 16
Source Project: sylph   Source File: JsonPathUdfTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void jsonPathUdfTestReturn123()
        throws Exception
{
    String jsonKey = "$.key1.key2";
    //Table table = tableEnv.sqlQuery("select cast(json['store'] as double) from tp , LATERAL TABLE(json_parser(message, 'store', 'ip')) as T(json) ");
    Table table1 = tableEnv.sqlQuery("select get_json_object(message,'" + jsonKey + "') from " + table);
    tableEnv.toAppendStream(table1, Row.class)
            .addSink(new SinkFunction<Row>()
            {
                @Override
                public void invoke(Row value, Context context)
                        throws Exception
                {
                    result.put(jsonKey, (String) value.getField(0));
                }
            });
    tableEnv.execute("");
    Assert.assertEquals("123", result.get(jsonKey));
}
 
Example 17
Source Project: flink-statefun   Source File: KafkaSinkProvider.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.setProperty(
        ProducerConfig.TRANSACTION_TIMEOUT_CONFIG,
        String.valueOf(spec.transactionTimeoutDuration().toMillis()));
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 
Example 18
Source Project: flink-statefun   Source File: EgressToSinkTranslator.java    License: Apache License 2.0 6 votes vote down vote up
private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 
Example 19
Source Project: Flink-CEPplus   Source File: Kafka011ITCase.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example 20
Source Project: flink   Source File: PubSubSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void invoke(IN message, SinkFunction.Context context) throws Exception {
	PubsubMessage pubsubMessage = PubsubMessage
		.newBuilder()
		.setData(ByteString.copyFrom(serializationSchema.serialize(message)))
		.build();

	ApiFuture<String> future = publisher.publish(pubsubMessage);
	numPendingFutures.incrementAndGet();
	ApiFutures.addCallback(future, failureHandler, directExecutor());
}
 
Example 21
Source Project: Flink-CEPplus   Source File: Kafka010ITCase.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example 22
Source Project: Flink-CEPplus   Source File: KafkaTableSinkBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void emitDataStream(DataStream<Row> dataStream) {
	final SinkFunction<Row> kafkaProducer = createKafkaProducer(
		topic,
		properties,
		serializationSchema,
		partitioner);
	dataStream.addSink(kafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 
Example 23
@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 
Example 24
Source Project: Flink-CEPplus   Source File: KeyedStream.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
	DataStreamSink<T> result = super.addSink(sinkFunction);
	result.getTransformation().setStateKeySelector(keySelector);
	result.getTransformation().setStateKeyType(keyType);
	return result;
}
 
Example 25
Source Project: flink   Source File: Kafka011ITCase.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example 26
Source Project: flink   Source File: KafkaITCase.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example 27
Source Project: flink   Source File: Kafka011TableSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected SinkFunction<Row> createKafkaProducer(
		String topic,
		Properties properties,
		SerializationSchema<Row> serializationSchema,
		Optional<FlinkKafkaPartitioner<Row>> partitioner) {
	return new FlinkKafkaProducer011<>(
		topic,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		properties,
		partitioner);
}
 
Example 28
Source Project: flink   Source File: Kafka011ITCase.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example 29
Source Project: flink   Source File: JobManagerWatermarkTrackerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 
Example 30
Source Project: flink   Source File: KeyedStream.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
	DataStreamSink<T> result = super.addSink(sinkFunction);
	result.getTransformation().setStateKeySelector(keySelector);
	result.getTransformation().setStateKeyType(keyType);
	return result;
}