Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#notifyOfCompletedCheckpoint()

The following examples show how to use org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#notifyOfCompletedCheckpoint() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 2
Source Project: flink   File: FlinkKafkaProducerITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 3
@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 4
Source Project: flink   File: WriteAheadSinkTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIdealCircumstances() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsIdealCircumstances(sink);
}
 
Example 5
@Test
public void testIdealCircumstances() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsIdealCircumstances(sink);
}
 
Example 6
Source Project: flink   File: BucketingSinkTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInactivityPeriodWithLateNotify() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness = createRescalingTestSink(outDir, 1, 0, 100);
	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new StreamRecord<>("test1", 1L));
	testHarness.processElement(new StreamRecord<>("test2", 1L));
	checkLocalFs(outDir, 2, 0 , 0, 0);

	testHarness.setProcessingTime(101L);	// put some in pending
	checkLocalFs(outDir, 0, 2, 0, 0);

	testHarness.snapshot(0, 0);				// put them in pending for 0
	checkLocalFs(outDir, 0, 2, 0, 0);

	testHarness.processElement(new StreamRecord<>("test3", 1L));
	testHarness.processElement(new StreamRecord<>("test4", 1L));

	testHarness.setProcessingTime(202L);	// put some in pending

	testHarness.snapshot(1, 0);				// put them in pending for 1
	checkLocalFs(outDir, 0, 4, 0, 0);

	testHarness.notifyOfCompletedCheckpoint(0);	// put the pending for 0 to the "committed" state
	checkLocalFs(outDir, 0, 2, 2, 0);

	testHarness.notifyOfCompletedCheckpoint(1); // put the pending for 1 to the "committed" state
	checkLocalFs(outDir, 0, 0, 4, 0);
}
 
Example 7
Source Project: flink   File: BucketingSinkTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRolloverInterval() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness = createRescalingTestSinkWithRollover(outDir, 1, 0, 1000L, 100L);
	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(0L);

	testHarness.processElement(new StreamRecord<>("test1", 1L));
	checkLocalFs(outDir, 1, 0, 0, 0);

	// invoke rollover based on rollover interval
	testHarness.setProcessingTime(101L);
	testHarness.processElement(new StreamRecord<>("test1", 2L));
	checkLocalFs(outDir, 1, 1, 0, 0);

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);
	checkLocalFs(outDir, 1, 0, 1, 0);

	// move the in-progress file to pending
	testHarness.setProcessingTime(3000L);
	testHarness.snapshot(1, 1);
	checkLocalFs(outDir, 0, 1, 1, 0);

	// move the pending file to "committed"
	testHarness.notifyOfCompletedCheckpoint(1);
	testHarness.close();

	checkLocalFs(outDir, 0, 0, 2, 0);
}
 
Example 8
Source Project: flink   File: WriteAheadSinkTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataPersistenceUponMissedNotify() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataPersistenceUponMissedNotify(sink);
}
 
Example 9
/**
 * This tests checks whether FlinkKafkaProducer011 correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 10
/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 11
Source Project: flink   File: GenericWriteAheadSinkTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
/**
 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
 * and later retry of the affected checkpoints.
 */
public void testCommitterException() throws Exception {

	ListSink2 sink = new ListSink2();

	OneInputStreamOperatorTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;

	for (int x = 0; x < 10; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);

	//isCommitted should have failed, thus sendValues() should never have been called
	Assert.assertEquals(0, sink.values.size());

	for (int x = 0; x < 11; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(1, 0);
	testHarness.notifyOfCompletedCheckpoint(1);

	//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
	Assert.assertEquals(10, sink.values.size());

	for (int x = 0; x < 12; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(2, 0);
	testHarness.notifyOfCompletedCheckpoint(2);

	//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 11 + 12 = 43 values
	Assert.assertEquals(43, sink.values.size());
}
 
Example 12
Source Project: flink   File: FlinkKafkaProducerITCase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 13
/**
 * This tests checks whether FlinkKafkaProducer011 correctly aborts lingering transactions after a failure,
 * which happened before first checkpoint and was followed up by reducing the parallelism.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testScaleDownBeforeFirstCheckpoint() throws Exception {
	String topic = "scale-down-before-first-checkpoint";

	List<AutoCloseable> operatorsToClose = new ArrayList<>();
	int preScaleDownParallelism = Math.max(2, FlinkKafkaProducer011.SAFE_SCALE_DOWN_FACTOR);
	for (int subtaskIndex = 0; subtaskIndex < preScaleDownParallelism; subtaskIndex++) {
		OneInputStreamOperatorTestHarness<Integer, Object> preScaleDownOperator = createTestHarness(
			topic,
			preScaleDownParallelism,
			preScaleDownParallelism,
			subtaskIndex,
			EXACTLY_ONCE);

		preScaleDownOperator.setup();
		preScaleDownOperator.open();
		preScaleDownOperator.processElement(subtaskIndex * 2, 0);
		preScaleDownOperator.snapshot(0, 1);
		preScaleDownOperator.processElement(subtaskIndex * 2 + 1, 2);

		operatorsToClose.add(preScaleDownOperator);
	}

	// do not close previous testHarnesses to make sure that closing do not clean up something (in case of failure
	// there might not be any close)

	// After previous failure simulate restarting application with smaller parallelism
	OneInputStreamOperatorTestHarness<Integer, Object> postScaleDownOperator1 = createTestHarness(topic, 1, 1, 0, EXACTLY_ONCE);

	postScaleDownOperator1.setup();
	postScaleDownOperator1.open();

	// write and commit more records, after potentially lingering transactions
	postScaleDownOperator1.processElement(46, 7);
	postScaleDownOperator1.snapshot(4, 8);
	postScaleDownOperator1.processElement(47, 9);
	postScaleDownOperator1.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42, 43, 44 and 45 in aborted transactions
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(46));

	postScaleDownOperator1.close();
	// ignore ProducerFencedExceptions, because postScaleDownOperator1 could reuse transactional ids.
	for (AutoCloseable operatorToClose : operatorsToClose) {
		closeIgnoringProducerFenced(operatorToClose);
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 14
Source Project: flink   File: FlinkKafkaProducerITCase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests checks whether FlinkKafkaProducer correctly aborts lingering transactions after a failure.
 * If such transactions were left alone lingering it consumers would be unable to read committed records
 * that were created after this lingering transaction.
 */
@Test
public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception {
	String topic = "flink-kafka-producer-fail-before-notify";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness1 = createTestHarness(topic);
	checkProducerLeak();
	testHarness1.setup();
	testHarness1.open();
	testHarness1.processElement(42, 0);
	testHarness1.snapshot(0, 1);
	testHarness1.processElement(43, 2);
	OperatorSubtaskState snapshot1 = testHarness1.snapshot(1, 3);

	testHarness1.processElement(44, 4);
	testHarness1.snapshot(2, 5);
	testHarness1.processElement(45, 6);

	// do not close previous testHarness to make sure that closing do not clean up something (in case of failure
	// there might not be any close)
	OneInputStreamOperatorTestHarness<Integer, Object> testHarness2 = createTestHarness(topic);
	testHarness2.setup();
	// restore from snapshot1, transactions with records 44 and 45 should be aborted
	testHarness2.initializeState(snapshot1);
	testHarness2.open();

	// write and commit more records, after potentially lingering transactions
	testHarness2.processElement(46, 7);
	testHarness2.snapshot(4, 8);
	testHarness2.processElement(47, 9);
	testHarness2.notifyOfCompletedCheckpoint(4);

	//now we should have:
	// - records 42 and 43 in committed transactions
	// - aborted transactions with records 44 and 45
	// - committed transaction with record 46
	// - pending transaction with record 47
	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42, 43, 46));

	try {
		testHarness1.close();
	} catch (Exception e) {
		// The only acceptable exception is ProducerFencedException because testHarness2 uses the same
		// transactional ID.
		if (!(e.getCause() instanceof ProducerFencedException)) {
			fail("Received unexpected exception " + e);
		}
	}
	testHarness2.close();
	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 15
Source Project: flink   File: WriteAheadSinkTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testDataDiscardingUponRestore() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	OperatorSubtaskState latestSnapshot = testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.close();

	sink = createSink();

	testHarness = new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.setup();
	testHarness.initializeState(latestSnapshot);
	testHarness.open();

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataDiscardingUponRestore(sink);
}
 
Example 16
Source Project: flink   File: BulkWriterTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testPartFilesWithIntegerBucketer(
		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
		File outDir,
		String partFileName1,
		String partFileName2,
		String partFileName3) throws Exception {

	testHarness.setup();
	testHarness.open();

	// this creates a new bucket "test1" and part-0-0
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
	TestUtils.checkLocalFs(outDir, 1, 0);

	// we take a checkpoint so we roll.
	testHarness.snapshot(1L, 1L);

	// these will close part-0-0 and open part-0-1 and part-0-2
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L));
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L));

	// we take a checkpoint so we roll again.
	testHarness.snapshot(2L, 2L);

	TestUtils.checkLocalFs(outDir, 3, 0);

	Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
	int fileCounter = 0;
	for (Map.Entry<File, String> fileContents : contents.entrySet()) {
		if (fileContents.getKey().getName().contains(partFileName1)) {
			fileCounter++;
			Assert.assertEquals("[email protected]\n", fileContents.getValue());
			Assert.assertEquals("1", fileContents.getKey().getParentFile().getName());
		} else if (fileContents.getKey().getName().contains(partFileName2)) {
			fileCounter++;
			Assert.assertEquals("[email protected]\n", fileContents.getValue());
			Assert.assertEquals("2", fileContents.getKey().getParentFile().getName());
		} else if (fileContents.getKey().getName().contains(partFileName3)) {
			fileCounter++;
			Assert.assertEquals("[email protected]\n", fileContents.getValue());
			Assert.assertEquals("3", fileContents.getKey().getParentFile().getName());
		}
	}
	Assert.assertEquals(3L, fileCounter);

	// we acknowledge the latest checkpoint, so everything should be published.
	testHarness.notifyOfCompletedCheckpoint(2L);

	TestUtils.checkLocalFs(outDir, 0, 3);
}
 
Example 17
@Test
public void testScalingDown() throws Exception {
	S sink1 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness1 =
		new OneInputStreamOperatorTestHarness<>(sink1, maxParallelism, 2, 0);
	testHarness1.open();

	S sink2 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness2 =
		new OneInputStreamOperatorTestHarness<>(sink2, maxParallelism, 2, 1);
	testHarness2.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 10; x++) {
		testHarness1.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	for (int x = 0; x < 11; x++) {
		testHarness2.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	// snapshot at checkpoint 0 for testHarness1 and testHarness 2
	OperatorSubtaskState snapshot1 = testHarness1.snapshot(snapshotCount, 0);
	OperatorSubtaskState snapshot2 = testHarness2.snapshot(snapshotCount, 0);

	// merge the two partial states
	OperatorSubtaskState mergedSnapshot = AbstractStreamOperatorTestHarness
		.repackageState(snapshot1, snapshot2);

	testHarness1.close();
	testHarness2.close();

	// and create a third instance that operates alone but
	// has the merged state of the previous 2 instances

	OperatorSubtaskState initState = AbstractStreamOperatorTestHarness.repartitionOperatorState(
		mergedSnapshot, maxParallelism, 2, 1, 0);

	S sink3 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> mergedTestHarness =
		new OneInputStreamOperatorTestHarness<>(sink3, maxParallelism, 1, 0);

	mergedTestHarness.setup();
	mergedTestHarness.initializeState(initState);
	mergedTestHarness.open();

	for (int x = 0; x < 12; x++) {
		mergedTestHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	snapshotCount++;
	mergedTestHarness.snapshot(snapshotCount, 1);
	mergedTestHarness.notifyOfCompletedCheckpoint(snapshotCount);

	verifyResultsWhenReScaling(sink3, 1, 33);
	mergedTestHarness.close();
}
 
Example 18
Source Project: flink   File: WriteAheadSinkTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testDataDiscardingUponRestore() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	OperatorSubtaskState latestSnapshot = testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.close();

	sink = createSink();

	testHarness = new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.setup();
	testHarness.initializeState(latestSnapshot);
	testHarness.open();

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsDataDiscardingUponRestore(sink);
}
 
Example 19
Source Project: flink   File: GenericWriteAheadSinkTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
/**
 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
 * and later retry of the affected checkpoints.
 */
public void testCommitterException() throws Exception {

	ListSink2 sink = new ListSink2();

	OneInputStreamOperatorTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;

	for (int x = 0; x < 10; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);

	//isCommitted should have failed, thus sendValues() should never have been called
	Assert.assertEquals(0, sink.values.size());

	for (int x = 0; x < 11; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(1, 0);
	testHarness.notifyOfCompletedCheckpoint(1);

	//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
	Assert.assertEquals(10, sink.values.size());

	for (int x = 0; x < 12; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(2, 0);
	testHarness.notifyOfCompletedCheckpoint(2);

	//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 11 + 12 = 43 values
	Assert.assertEquals(43, sink.values.size());
}
 
Example 20
Source Project: flink   File: BulkWriterTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testPartFiles(
		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
		File outDir,
		String partFileName1,
		String partFileName2) throws Exception {

	testHarness.setup();
	testHarness.open();

	// this creates a new bucket "test1" and part-0-0
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
	TestUtils.checkLocalFs(outDir, 1, 0);

	// we take a checkpoint so we roll.
	testHarness.snapshot(1L, 1L);

	// these will close part-0-0 and open part-0-1
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L));
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L));

	// we take a checkpoint so we roll again.
	testHarness.snapshot(2L, 2L);

	TestUtils.checkLocalFs(outDir, 2, 0);

	Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
	int fileCounter = 0;
	for (Map.Entry<File, String> fileContents : contents.entrySet()) {
		if (fileContents.getKey().getName().contains(partFileName1)) {
			fileCounter++;
			Assert.assertEquals("[email protected]\n", fileContents.getValue());
		} else if (fileContents.getKey().getName().contains(partFileName2)) {
			fileCounter++;
			Assert.assertEquals("[email protected]\[email protected]\n", fileContents.getValue());
		}
	}
	Assert.assertEquals(2L, fileCounter);

	// we acknowledge the latest checkpoint, so everything should be published.
	testHarness.notifyOfCompletedCheckpoint(2L);

	TestUtils.checkLocalFs(outDir, 0, 2);
}