Java Code Examples for org.apache.flink.runtime.operators.testutils.UniformRecordGenerator

The following examples show how to use org.apache.flink.runtime.operators.testutils.UniformRecordGenerator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: ReduceTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FlatMapTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMapTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
	setOutput(this.output);
	
	final FlatMapDriver<Record, Record> testDriver = new FlatMapDriver<>();
	
	try {
		testDriver(testDriver, MockMapStub.class);
	} catch (Exception e) {
		LOG.debug("Exception while running the test driver.", e);
		Assert.fail("Invoke method caused exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", keyCnt*valCnt, this.output.getNumberOfRecords());
}
 
Example 3
Source Project: Flink-CEPplus   Source File: ReduceTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 
Example 4
Source Project: Flink-CEPplus   Source File: CoGroupTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth1CoGroupTask() {
	int keyCnt1 = 100;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 1;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 5
Source Project: Flink-CEPplus   Source File: CoGroupTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth2CoGroupTask() {
	int keyCnt1 = 200;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 4;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 6
Source Project: Flink-CEPplus   Source File: CoGroupTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortSecondCoGroupTask() {
	int keyCnt1 = 200;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 4;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 7
Source Project: Flink-CEPplus   Source File: CoGroupTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMergeCoGroupTask() {
	int keyCnt1 = 200;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 4;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 8
Source Project: Flink-CEPplus   Source File: CoGroupTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingSortCoGroupTask() {
	int keyCnt1 = 100;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 1;
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		testDriver(testTask, MockFailingCoGroupStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: CachedMatchTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash1MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 10;
	int valCnt2 = 2;
			
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 10
Source Project: flink   Source File: CrossTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlockEmptyOuterCrossTask() {
	int keyCnt1 = 10;
	int valCnt1 = 1;
	
	int keyCnt2 = 0;
	int valCnt2 = 0;
	
	final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
			
	getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND);
	getTaskConfig().setRelativeMemoryDriver(cross_frac);
	
	final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
	
	try {
		testDriver(testTask, MockCrossStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test failed due to an exception.");
	}
	
	Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 11
Source Project: Flink-CEPplus   Source File: CachedMatchTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash3MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 12
Source Project: flink   Source File: DataSinkTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingDataSinkTask() {

	int keyCnt = 100;
	int valCnt = 20;

	super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	super.addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);

	DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
	Configuration stubParams = new Configuration();

	File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
	super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString(), stubParams);

	boolean stubFailed = false;

	try {
		testTask.invoke();
	} catch (Exception e) {
		stubFailed = true;
	}
	Assert.assertTrue("Function exception was not forwarded.", stubFailed);

	// assert that temp file was removed
	Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());

}
 
Example 13
Source Project: Flink-CEPplus   Source File: CachedMatchTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 14
Source Project: Flink-CEPplus   Source File: CachedMatchTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: CachedMatchTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingHashSecondMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth1MatchTask() {
	final int keyCnt1 = 20;
	final int valCnt1 = 1;
	
	final int keyCnt2 = 10;
	final int valCnt2 = 2;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
 
Example 17
Source Project: flink   Source File: CrossTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStream1CrossTask() {
	int keyCnt1 = 10;
	int valCnt1 = 1;
	
	int keyCnt2 = 100;
	int valCnt2 = 4;
	
	final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
			
	getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST);
	getTaskConfig().setRelativeMemoryDriver(cross_frac);
	
	final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
	
	try {
		testDriver(testTask, MockCrossStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test failed due to an exception.");
	}
	
	Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
	
}
 
Example 18
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth3MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 
Example 19
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth4MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 
Example 20
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortBoth5MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 
Example 21
Source Project: flink   Source File: CrossTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlockEmptyInnerCrossTask() {
	int keyCnt1 = 10;
	int valCnt1 = 1;
	
	int keyCnt2 = 0;
	int valCnt2 = 0;
	
	final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
			
	getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST);
	getTaskConfig().setRelativeMemoryDriver(cross_frac);
	
	final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
	
	try {
		testDriver(testTask, MockCrossStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test failed due to an exception.");
	}
	
	Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 22
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortSecondMatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 
Example 23
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMergeMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
 
Example 24
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash1MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 10;
	int valCnt2 = 2;
			
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 25
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash2MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 26
Source Project: flink   Source File: ReduceTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReduceTaskOnPreSortedInput() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for(Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
}
 
Example 27
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash4MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 28
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
 
Example 29
Source Project: flink   Source File: JoinTaskExternalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testExternalHash1MatchTask() {
	final int keyCnt1 = 32768;
	final int valCnt1 = 8;
	
	final int keyCnt2 = 65536;
	final int valCnt2 = 8;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.output);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example 30
Source Project: Flink-CEPplus   Source File: JoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingHashSecondMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockFailingMatchStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}