org.apache.flink.streaming.api.functions.sink.SinkFunction Java Examples

The following examples show how to use org.apache.flink.streaming.api.functions.sink.SinkFunction. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ElasticsearchUpsertTableSinkBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DataStreamSink<?> consumeDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	return dataStream.addSink(sinkFunction)
		.setParallelism(dataStream.getParallelism())
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 
Example #2
Source File: ElasticsearchUpsertTableSinkBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
	final ElasticsearchUpsertSinkFunction upsertFunction =
		new ElasticsearchUpsertSinkFunction(
			index,
			docType,
			keyDelimiter,
			keyNullLiteral,
			serializationSchema,
			contentType,
			requestFactory,
			keyFieldIndices);
	final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
		hosts,
		failureHandler,
		sinkOptions,
		upsertFunction);
	dataStream.addSink(sinkFunction)
		.name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));
}
 
Example #3
Source File: KafkaSinkProvider.java    From flink-statefun with Apache License 2.0 6 votes vote down vote up
@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.setProperty(
        ProducerConfig.TRANSACTION_TIMEOUT_CONFIG,
        String.valueOf(spec.transactionTimeoutDuration().toMillis()));
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 
Example #4
Source File: GroupedProcessingTimeWindowExample.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 
Example #5
Source File: EgressToSinkTranslator.java    From flink-statefun with Apache License 2.0 6 votes vote down vote up
private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 
Example #6
Source File: JsonPathUdfTest.java    From sylph with Apache License 2.0 6 votes vote down vote up
@Test
public void jsonPathUdfTestReturn123()
        throws Exception
{
    String jsonKey = "$.key1.key2";
    //Table table = tableEnv.sqlQuery("select cast(json['store'] as double) from tp , LATERAL TABLE(json_parser(message, 'store', 'ip')) as T(json) ");
    Table table1 = tableEnv.sqlQuery("select get_json_object(message,'" + jsonKey + "') from " + table);
    tableEnv.toAppendStream(table1, Row.class)
            .addSink(new SinkFunction<Row>()
            {
                @Override
                public void invoke(Row value, Context context)
                        throws Exception
                {
                    result.put(jsonKey, (String) value.getField(0));
                }
            });
    tableEnv.execute("");
    Assert.assertEquals("123", result.get(jsonKey));
}
 
Example #7
Source File: DataStream.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example #8
Source File: StreamCheckpointingITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Runs the following program.
 * <pre>
 *     [ (source)->(filter) ]-s->[ (map) ] -> [ (map) ] -> [ (groupBy/count)->(sink) ]
 * </pre>
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {
	DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS));

	stream
			// -------------- first vertex, chained to the source ----------------
			.filter(new StringRichFilterFunction())
			.shuffle()

			// -------------- seconds vertex - the stateful one that also fails ----------------
			.map(new StringPrefixCountRichMapFunction())
			.startNewChain()
			.map(new StatefulCounterFunction())

			// -------------- third vertex - counter and the sink ----------------
			.keyBy("prefix")
			.map(new OnceFailingPrefixCounter(NUM_STRINGS))
			.addSink(new SinkFunction<PrefixCount>() {

				@Override
				public void invoke(PrefixCount value) throws Exception {
					// Do nothing here
				}
			});
}
 
Example #9
Source File: Elasticsearch6TableSink.java    From alchemy with Apache License 2.0 6 votes vote down vote up
@Override
public void emitDataStream(DataStream<Tuple2<Boolean, Row>> dataStream) {
    Elasticsearch6SinkFunction upsertFunction = new Elasticsearch6SinkFunction(
            index,
            findIndex(indexField, schema.getFieldNames()),
            docType,
            keyDelimiter,
            keyNullLiteral,
            serializationSchema,
            contentType,
            requestFactory,
            keyFieldIndices);
    final SinkFunction<Tuple2<Boolean, Row>> sinkFunction = createSinkFunction(
            hosts,
            failureHandler,
            upsertFunction);
    dataStream.addSink(sinkFunction)
            .name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames()));

}
 
Example #10
Source File: EgressToSinkTranslator.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up
private DecoratedSink sinkFromSpec(EgressIdentifier<?> key, EgressSpec<?> spec) {
  SinkProvider provider = universe.sinks().get(spec.type());
  if (provider == null) {
    throw new IllegalStateException(
        "Unable to find a sink translation for egress of type "
            + spec.type()
            + ", which is bound for key "
            + key);
  }
  SinkFunction<?> sink = provider.forSpec(spec);
  if (sink == null) {
    throw new NullPointerException(
        "A sink provider for type " + spec.type() + ", has produced a NULL sink.");
  }
  return DecoratedSink.of(spec, sink);
}
 
Example #11
Source File: StreamGraphGeneratorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 
Example #12
Source File: DataStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example #13
Source File: Buckets.java    From flink with Apache License 2.0 6 votes vote down vote up
Bucket<IN, BucketID> onElement(final IN value, final SinkFunction.Context context) throws Exception {
	final long currentProcessingTime = context.currentProcessingTime();

	// setting the values in the bucketer context
	bucketerContext.update(
			context.timestamp(),
			context.currentWatermark(),
			currentProcessingTime);

	final BucketID bucketId = bucketAssigner.getBucketId(value, bucketerContext);
	final Bucket<IN, BucketID> bucket = getOrCreateBucketForBucketId(bucketId);
	bucket.write(value, currentProcessingTime);

	// we update the global max counter here because as buckets become inactive and
	// get removed from the list of active buckets, at the time when we want to create
	// another part file for the bucket, if we start from 0 we may overwrite previous parts.

	this.maxPartCounter = Math.max(maxPartCounter, bucket.getPartCounter());
	return bucket;
}
 
Example #14
Source File: GroupedProcessingTimeWindowExample.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(4);

		DataStream<Tuple2<Long, Long>> stream = env.addSource(new DataSource());

		stream
			.keyBy(0)
			.timeWindow(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
			.reduce(new SummingReducer())

			// alternative: use a apply function which does not pre-aggregate
//			.keyBy(new FirstFieldKeyExtractor<Tuple2<Long, Long>, Long>())
//			.window(Time.of(2500, MILLISECONDS), Time.of(500, MILLISECONDS))
//			.apply(new SummingWindowFunction())

			.addSink(new SinkFunction<Tuple2<Long, Long>>() {
				@Override
				public void invoke(Tuple2<Long, Long> value) {
				}
			});

		env.execute();
	}
 
Example #15
Source File: DataStream.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 
Example #16
Source File: ActiveMQConnectorITCase.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
private void createConsumerTopology(StreamExecutionEnvironment env, AMQSourceConfig<String> config) {
    AMQSource<String> source = new AMQSource<>(config);

    env.addSource(source)
        .addSink(new SinkFunction<String>() {
            final HashSet<Integer> set = new HashSet<>();
            @Override
            public void invoke(String value, Context context) throws Exception {
                int val = Integer.parseInt(value.split("-")[1]);
                set.add(val);

                if (set.size() == MESSAGES_NUM) {
                    throw new SuccessException();
                }
            }
        });
}
 
Example #17
Source File: StreamGraphGeneratorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the json generated by JSONGenerator shall meet with 2 requirements:
 * 1. sink nodes are at the back
 * 2. if both two nodes are sink nodes or neither of them is sink node, then sort by its id.
 */
@Test
public void testSinkIdComparison() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> source = env.fromElements(1, 2, 3);
	for (int i = 0; i < 32; i++) {
		if (i % 2 == 0) {
			source.addSink(new SinkFunction<Integer>() {
				@Override
				public void invoke(Integer value, Context ctx) throws Exception {}
			});
		} else {
			source.map(x -> x + 1);
		}
	}
	// IllegalArgumentException will be thrown without FLINK-9216
	env.getStreamGraph().getStreamingPlanAsJSON();
}
 
Example #18
Source File: KafkaSinkProvider.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up
@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> egressSpec) {
  KafkaEgressSpec<T> spec = asSpec(egressSpec);

  Properties properties = new Properties();
  properties.putAll(spec.properties());
  properties.put("bootstrap.servers", spec.kafkaAddress());

  Semantic producerSemantic = semanticFromSpec(spec);
  if (producerSemantic == Semantic.EXACTLY_ONCE) {
    properties.put("transaction.timeout.ms", spec.transactionTimeoutDuration().toMillis());
  }

  return new FlinkKafkaProducer<>(
      randomKafkaTopic(),
      serializerFromSpec(spec),
      properties,
      producerSemantic,
      spec.kafkaProducerPoolSize());
}
 
Example #19
Source File: SinkOperator.java    From flink with Apache License 2.0 5 votes vote down vote up
public SinkOperator(
		SinkFunction<RowData> sinkFunction,
		int rowtimeFieldIndex,
		NotNullEnforcer notNullEnforcer,
		int[] notNullFieldIndices,
		String[] allFieldNames) {
	super(sinkFunction);
	this.rowtimeFieldIndex = rowtimeFieldIndex;
	this.notNullFieldIndices = notNullFieldIndices;
	this.notNullEnforcer = notNullEnforcer;
	this.notNullCheck = notNullFieldIndices.length > 0;
	this.allFieldNames = allFieldNames;
	chainingStrategy = ChainingStrategy.ALWAYS;
}
 
Example #20
Source File: HeapMonitorPipelineTest.java    From flink-tutorials with Apache License 2.0 5 votes vote down vote up
@Test
public void testPipeline() throws Exception {

	final String alertMask = "42";

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	HeapMetrics alert1 = testStats(0.42);
	HeapMetrics regular1 = testStats(0.452);
	HeapMetrics regular2 = testStats(0.245);
	HeapMetrics alert2 = testStats(0.9423);

	DataStreamSource<HeapMetrics> testInput = env.fromElements(alert1, alert2, regular1, regular2);
	HeapMonitorPipeline.computeHeapAlerts(testInput, ParameterTool.fromArgs(new String[]{"--alertMask", alertMask}))
		.addSink(new SinkFunction<HeapAlert>() {
			@Override
			public void invoke(HeapAlert value, Context context) {
				testOutput.add(value);
			}
		})
		.setParallelism(1);

	env.execute();

	assertEquals(Sets.newHashSet(HeapAlert.maskRatioMatch(alertMask, alert1),
			HeapAlert.maskRatioMatch(alertMask, alert2)), testOutput);
}
 
Example #21
Source File: Kafka011TableSink.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected SinkFunction<Row> createKafkaProducer(
		String topic,
		Properties properties,
		SerializationSchema<Row> serializationSchema,
		Optional<FlinkKafkaPartitioner<Row>> partitioner) {
	return new FlinkKafkaProducer011<>(
		topic,
		serializationSchema,
		properties,
		partitioner);
}
 
Example #22
Source File: JobManagerWatermarkTrackerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 
Example #23
Source File: DecoratedSink.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
public static DecoratedSink of(EgressSpec<?> spec, SinkFunction<?> sink) {
  EgressIdentifier<?> identifier = spec.id();
  String name = String.format("%s-%s-egress", identifier.namespace(), identifier.name());
  String uid =
      String.format(
          "%s-%s-%s-%s-egress",
          spec.type().namespace(), spec.type().type(), identifier.namespace(), identifier.name());

  return new DecoratedSink(name, uid, sink);
}
 
Example #24
Source File: SourceSinkModule.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
@Override
public <T> SinkFunction<T> forSpec(EgressSpec<T> spec) {
  if (!(spec instanceof SinkFunctionSpec)) {
    throw new IllegalStateException("spec " + spec + " is not of type SourceFunctionSpec");
  }
  SinkFunctionSpec<T> casted = (SinkFunctionSpec<T>) spec;
  return casted.delegate();
}
 
Example #25
Source File: HarnessIoModule.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
private static <T> SinkFunction<T> consumingEgressSpec(EgressSpec<T> spec) {
  if (!(spec instanceof ConsumingEgressSpec)) {
    throw new IllegalArgumentException("Unable to provider a source for " + spec);
  }
  ConsumingEgressSpec<T> casted = (ConsumingEgressSpec<T>) spec;
  return new ConsumingSink<>(casted.consumer());
}
 
Example #26
Source File: KinesisSinkProviderTest.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
@Test
public void exampleUsage() {
  final KinesisEgressSpec<String> kinesisEgressSpec =
      KinesisEgressBuilder.forIdentifier(ID)
          .withAwsRegion("us-west-1")
          .withAwsCredentials(AwsCredentials.basic("access-key-id", "secret-access-key"))
          .withSerializer(TestSerializer.class)
          .build();

  final KinesisSinkProvider provider = new KinesisSinkProvider();
  final SinkFunction<String> sink = provider.forSpec(kinesisEgressSpec);

  assertThat(sink, instanceOf(FlinkKinesisProducer.class));
}
 
Example #27
Source File: KeyedStream.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {
	DataStreamSink<T> result = super.addSink(sinkFunction);
	result.getTransformation().setStateKeySelector(keySelector);
	result.getTransformation().setStateKeyType(keyType);
	return result;
}
 
Example #28
Source File: KafkaITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
public TimestampValidatingOperator() {
	super(new SinkFunction<Long>() {
		private static final long serialVersionUID = -6676565693361786524L;

		@Override
		public void invoke(Long value) throws Exception {
			throw new RuntimeException("Unexpected");
		}
	});
}
 
Example #29
Source File: JobManagerWatermarkTrackerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}
 
Example #30
Source File: TranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static StreamExecutionEnvironment getSimpleJob() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.generateSequence(1, 10000000)
			.addSink(new SinkFunction<Long>() {
				@Override
				public void invoke(Long value) {
				}
			});

	return env;
}