Java Code Examples for org.apache.flink.streaming.connectors.kafka.testutils.FailingIdentityMapper

The following examples show how to use org.apache.flink.streaming.connectors.kafka.testutils.FailingIdentityMapper. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: KafkaShuffleExactlyOnceITCase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * To test failure recovery after processing 2/3 data.
 *
 * <p>Schema: (key, timestamp, source instance Id).
 * Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1
 */
private void testKafkaShuffleFailureRecovery(
		int numElementsPerProducer,
		TimeCharacteristic timeCharacteristic) throws Exception {

	String topic = topic("failure_recovery", timeCharacteristic);
	final int numberOfPartitions = 1;
	final int producerParallelism = 1;
	final int failAfterElements = numElementsPerProducer * numberOfPartitions * 2 / 3;

	createTestTopic(topic, numberOfPartitions, 1);

	final StreamExecutionEnvironment env =
		createEnvironment(producerParallelism, timeCharacteristic).enableCheckpointing(500);

	createKafkaShuffle(
		env, topic, numElementsPerProducer, producerParallelism, timeCharacteristic, numberOfPartitions)
		.map(new FailingIdentityMapper<>(failAfterElements)).setParallelism(1)
		.map(new ToInteger(producerParallelism)).setParallelism(1)
		.addSink(new ValidatingExactlyOnceSink(numElementsPerProducer * producerParallelism)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;

	tryExecute(env, topic);

	deleteTestTopic(topic);
}
 
Example 2
Source Project: flink   Source File: KafkaShuffleExactlyOnceITCase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * To test failure recovery with partition assignment after processing 2/3 data.
 *
 * <p>Schema: (key, timestamp, source instance Id).
 * Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3
 */
private void testAssignedToPartitionFailureRecovery(
		int numElementsPerProducer,
		TimeCharacteristic timeCharacteristic) throws Exception {
	String topic = topic("partition_failure_recovery", timeCharacteristic);
	final int numberOfPartitions = 3;
	final int producerParallelism = 2;
	final int failAfterElements = numElementsPerProducer * producerParallelism * 2 / 3;

	createTestTopic(topic, numberOfPartitions, 1);

	final StreamExecutionEnvironment env = createEnvironment(producerParallelism, timeCharacteristic);

	KeyedStream<Tuple3<Integer, Long, Integer>, Tuple> keyedStream = createKafkaShuffle(
		env,
		topic,
		numElementsPerProducer,
		producerParallelism,
		timeCharacteristic,
		numberOfPartitions);
	keyedStream
		.process(new PartitionValidator(keyedStream.getKeySelector(), numberOfPartitions, topic))
		.setParallelism(numberOfPartitions)
		.map(new ToInteger(producerParallelism)).setParallelism(numberOfPartitions)
		.map(new FailingIdentityMapper<>(failAfterElements)).setParallelism(1)
		.addSink(new ValidatingExactlyOnceSink(numElementsPerProducer * producerParallelism)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;

	tryExecute(env, topic);

	deleteTestTopic(topic);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: KafkaProducerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This test sets KafkaProducer so that it will  automatically flush the data and
 * and fails the broker to check whether flushed records since last checkpoint were not duplicated.
 */
protected void testExactlyOnce(boolean regularSink, int sinksCount) throws Exception {
	final String topic = (regularSink ? "exactlyOnceTopicRegularSink" : "exactlyTopicCustomOperator") + sinksCount;
	final int partition = 0;
	final int numElements = 1000;
	final int failAfterElements = 333;

	for (int i = 0; i < sinksCount; i++) {
		createTestTopic(topic + i, 1, 1);
	}

	TypeInformationSerializationSchema<Integer> schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
	KeyedSerializationSchema<Integer> keyedSerializationSchema = new KeyedSerializationSchemaWrapper<>(schema);

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties properties = new Properties();
	properties.putAll(standardProps);
	properties.putAll(secureProps);

	// process exactly failAfterElements number of elements and then shutdown Kafka broker and fail application
	List<Integer> expectedElements = getIntegersSequence(numElements);

	DataStream<Integer> inputStream = env
		.addSource(new IntegerSource(numElements))
		.map(new FailingIdentityMapper<Integer>(failAfterElements));

	for (int i = 0; i < sinksCount; i++) {
		FlinkKafkaPartitioner<Integer> partitioner = new FlinkKafkaPartitioner<Integer>() {
			@Override
			public int partition(Integer record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
				return partition;
			}
		};

		if (regularSink) {
			StreamSink<Integer> kafkaSink = kafkaServer.getProducerSink(topic + i, keyedSerializationSchema, properties, partitioner);
			inputStream.addSink(kafkaSink.getUserFunction());
		} else {
			kafkaServer.produceIntoKafka(inputStream, topic + i, keyedSerializationSchema, properties, partitioner);
		}
	}

	FailingIdentityMapper.failedBefore = false;
	TestUtils.tryExecute(env, "Exactly once test");

	for (int i = 0; i < sinksCount; i++) {
		// assert that before failure we successfully snapshot/flushed all expected elements
		assertExactlyOnceForTopic(
			properties,
			topic + i,
			partition,
			expectedElements,
			KAFKA_READ_TIMEOUT);
		deleteTestTopic(topic + i);
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having a 1:1 correspondence between kafka partitions and
 * Flink sources.
 */
public void runOneToOneExactlyOnceTest() throws Exception {

	final String topic = "oneToOneTopic";
	final int parallelism = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = parallelism * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	createTestTopic(topic, parallelism, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			parallelism,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);

	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(parallelism, 1))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-to-one exactly once test");

	deleteTestTopic(topic);
}
 
Example 5
Source Project: Flink-CEPplus   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having fewer Flink sources than Kafka partitions, so
 * one Flink source will read multiple Kafka partitions.
 */
public void runOneSourceMultiplePartitionsExactlyOnceTest() throws Exception {
	final String topic = "oneToManyTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 2;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(numPartitions, 3))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-source-multi-partitions exactly once test");

	deleteTestTopic(topic);
}
 
Example 6
Source Project: Flink-CEPplus   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having more Flink sources than Kafka partitions, which means
 * that some Flink sources will read no partitions.
 */
public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
	final String topic = "manyToOneTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 8;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	// set the number of restarts to one. The failing mapper will fail once, then it's only success exceptions.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();
	env.setBufferTimeout(0);

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
		.addSource(kafkaSource)
		.map(new PartitionValidatingMapper(numPartitions, 1))
		.map(new FailingIdentityMapper<Integer>(failAfterElements))
		.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "multi-source-one-partitions exactly once test");

	deleteTestTopic(topic);
}
 
Example 7
Source Project: flink   Source File: KafkaProducerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This test sets KafkaProducer so that it will  automatically flush the data and
 * and fails the broker to check whether flushed records since last checkpoint were not duplicated.
 */
protected void testExactlyOnce(boolean regularSink, int sinksCount) throws Exception {
	final String topic = (regularSink ? "exactlyOnceTopicRegularSink" : "exactlyTopicCustomOperator") + sinksCount;
	final int partition = 0;
	final int numElements = 1000;
	final int failAfterElements = 333;

	for (int i = 0; i < sinksCount; i++) {
		createTestTopic(topic + i, 1, 1);
	}

	TypeInformationSerializationSchema<Integer> schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
	KeyedSerializationSchema<Integer> keyedSerializationSchema = new KeyedSerializationSchemaWrapper<>(schema);

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties properties = new Properties();
	properties.putAll(standardProps);
	properties.putAll(secureProps);

	// process exactly failAfterElements number of elements and then shutdown Kafka broker and fail application
	List<Integer> expectedElements = getIntegersSequence(numElements);

	DataStream<Integer> inputStream = env
		.addSource(new IntegerSource(numElements))
		.map(new FailingIdentityMapper<Integer>(failAfterElements));

	for (int i = 0; i < sinksCount; i++) {
		FlinkKafkaPartitioner<Integer> partitioner = new FlinkKafkaPartitioner<Integer>() {
			@Override
			public int partition(Integer record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
				return partition;
			}
		};

		if (regularSink) {
			StreamSink<Integer> kafkaSink = kafkaServer.getProducerSink(topic + i, keyedSerializationSchema, properties, partitioner);
			inputStream.addSink(kafkaSink.getUserFunction());
		} else {
			kafkaServer.produceIntoKafka(inputStream, topic + i, keyedSerializationSchema, properties, partitioner);
		}
	}

	FailingIdentityMapper.failedBefore = false;
	TestUtils.tryExecute(env, "Exactly once test");

	for (int i = 0; i < sinksCount; i++) {
		// assert that before failure we successfully snapshot/flushed all expected elements
		assertExactlyOnceForTopic(
			properties,
			topic + i,
			partition,
			expectedElements,
			KAFKA_READ_TIMEOUT);
		deleteTestTopic(topic + i);
	}
}
 
Example 8
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having a 1:1 correspondence between kafka partitions and
 * Flink sources.
 */
public void runOneToOneExactlyOnceTest() throws Exception {

	final String topic = "oneToOneTopic";
	final int parallelism = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = parallelism * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	createTestTopic(topic, parallelism, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			parallelism,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);

	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(parallelism, 1))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-to-one exactly once test");

	deleteTestTopic(topic);
}
 
Example 9
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having fewer Flink sources than Kafka partitions, so
 * one Flink source will read multiple Kafka partitions.
 */
public void runOneSourceMultiplePartitionsExactlyOnceTest() throws Exception {
	final String topic = "oneToManyTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 2;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(numPartitions, 3))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-source-multi-partitions exactly once test");

	deleteTestTopic(topic);
}
 
Example 10
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having more Flink sources than Kafka partitions, which means
 * that some Flink sources will read no partitions.
 */
public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
	final String topic = "manyToOneTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 8;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	// set the number of restarts to one. The failing mapper will fail once, then it's only success exceptions.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
	env.getConfig().disableSysoutLogging();
	env.setBufferTimeout(0);

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
		.addSource(kafkaSource)
		.map(new PartitionValidatingMapper(numPartitions, 1))
		.map(new FailingIdentityMapper<Integer>(failAfterElements))
		.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "multi-source-one-partitions exactly once test");

	deleteTestTopic(topic);
}
 
Example 11
Source Project: flink   Source File: KafkaProducerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This test sets KafkaProducer so that it will  automatically flush the data and
 * and fails the broker to check whether flushed records since last checkpoint were not duplicated.
 */
protected void testExactlyOnce(boolean regularSink, int sinksCount) throws Exception {
	final String topic = (regularSink ? "exactlyOnceTopicRegularSink" : "exactlyTopicCustomOperator") + sinksCount;
	final int partition = 0;
	final int numElements = 1000;
	final int failAfterElements = 333;

	for (int i = 0; i < sinksCount; i++) {
		createTestTopic(topic + i, 1, 1);
	}

	TypeInformationSerializationSchema<Integer> schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(1);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));

	Properties properties = new Properties();
	properties.putAll(standardProps);
	properties.putAll(secureProps);

	// process exactly failAfterElements number of elements and then shutdown Kafka broker and fail application
	List<Integer> expectedElements = getIntegersSequence(numElements);

	DataStream<Integer> inputStream = env
		.addSource(new IntegerSource(numElements))
		.map(new FailingIdentityMapper<Integer>(failAfterElements));

	for (int i = 0; i < sinksCount; i++) {
		FlinkKafkaPartitioner<Integer> partitioner = new FlinkKafkaPartitioner<Integer>() {
			@Override
			public int partition(Integer record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
				return partition;
			}
		};

		if (regularSink) {
			StreamSink<Integer> kafkaSink = kafkaServer.getProducerSink(topic + i, schema, properties, partitioner);
			inputStream.addSink(kafkaSink.getUserFunction());
		} else {
			kafkaServer.produceIntoKafka(inputStream, topic + i, schema, properties, partitioner);
		}
	}

	FailingIdentityMapper.failedBefore = false;
	TestUtils.tryExecute(env, "Exactly once test");

	for (int i = 0; i < sinksCount; i++) {
		// assert that before failure we successfully snapshot/flushed all expected elements
		assertExactlyOnceForTopic(
			properties,
			topic + i,
			partition,
			expectedElements,
			KAFKA_READ_TIMEOUT);
		deleteTestTopic(topic + i);
	}
}
 
Example 12
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having a 1:1 correspondence between kafka partitions and
 * Flink sources.
 */
public void runOneToOneExactlyOnceTest() throws Exception {

	final String topic = "oneToOneTopic";
	final int parallelism = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = parallelism * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	createTestTopic(topic, parallelism, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			parallelism,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);

	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(parallelism, 1))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-to-one exactly once test");

	deleteTestTopic(topic);
}
 
Example 13
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having fewer Flink sources than Kafka partitions, so
 * one Flink source will read multiple Kafka partitions.
 */
public void runOneSourceMultiplePartitionsExactlyOnceTest() throws Exception {
	final String topic = "oneToManyTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 2;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
			.addSource(kafkaSource)
			.map(new PartitionValidatingMapper(numPartitions, 3))
			.map(new FailingIdentityMapper<Integer>(failAfterElements))
			.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "One-source-multi-partitions exactly once test");

	deleteTestTopic(topic);
}
 
Example 14
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests the proper consumption when having more Flink sources than Kafka partitions, which means
 * that some Flink sources will read no partitions.
 */
public void runMultipleSourcesOnePartitionExactlyOnceTest() throws Exception {
	final String topic = "manyToOneTopic";
	final int numPartitions = 5;
	final int numElementsPerPartition = 1000;
	final int totalElements = numPartitions * numElementsPerPartition;
	final int failAfterElements = numElementsPerPartition / 3;

	final int parallelism = 8;

	createTestTopic(topic, numPartitions, 1);

	DataGenerators.generateRandomizedIntegerSequence(
			StreamExecutionEnvironment.getExecutionEnvironment(),
			kafkaServer,
			topic,
			numPartitions,
			numElementsPerPartition,
			true);

	// run the topology that fails and recovers

	DeserializationSchema<Integer> schema =
			new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setParallelism(parallelism);
	// set the number of restarts to one. The failing mapper will fail once, then it's only success exceptions.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
			env.setBufferTimeout(0);

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<Integer> kafkaSource = kafkaServer.getConsumer(topic, schema, props);

	env
		.addSource(kafkaSource)
		.map(new PartitionValidatingMapper(numPartitions, 1))
		.map(new FailingIdentityMapper<Integer>(failAfterElements))
		.addSink(new ValidatingExactlyOnceSink(totalElements)).setParallelism(1);

	FailingIdentityMapper.failedBefore = false;
	tryExecute(env, "multi-source-one-partitions exactly once test");

	deleteTestTopic(topic);
}