Java Code Examples for org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator

The following examples show how to use org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: LeftOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: RightOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #8
Source Project: Flink-CEPplus   Author: ljygz   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: AbstractOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: LeftOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: RightOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #19
Source Project: flink   Author: apache   File: AbstractOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #20
Source Project: flink   Author: apache   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #21
Source Project: flink   Author: apache   File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #22
Source Project: flink   Author: apache   File: LeftOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #23
Source Project: flink   Author: apache   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
 
Example #24
Source Project: flink   Author: apache   File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #25
Source Project: flink   Author: apache   File: RightOuterJoinTaskExternalITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
 
Example #26
Source Project: flink   Author: apache   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
 
Example #27
Source Project: flink   Author: apache   File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: CombineTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCombineTask() {
	try {
		int keyCnt = 100;
		int valCnt = 20;
		
		setInput(new UniformIntTupleGenerator(keyCnt, valCnt, false), serializer);
		addDriverComparator(this.comparator);
		addDriverComparator(this.comparator);
		setOutput(this.outList, serializer);

		getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
		getTaskConfig().setRelativeMemoryDriver(combine_frac);
		getTaskConfig().setFilehandlesDriver(2);
		
		final GroupReduceCombineDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask =
				new GroupReduceCombineDriver<>();
		
		testDriver(testTask, MockCombiningReduceStub.class);
		
		int expSum = 0;
		for (int i = 1;i < valCnt; i++) {
			expSum += i;
		}
		
		assertTrue(this.outList.size() == keyCnt);
		
		for (Tuple2<Integer, Integer> record : this.outList) {
			assertTrue(record.f1 == expSum);
		}
		
		this.outList.clear();
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: CombineTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingCombineTask() {
	try {
		int keyCnt = 100;
		int valCnt = 20;
		
		setInput(new UniformIntTupleGenerator(keyCnt, valCnt, false), serializer);
		addDriverComparator(this.comparator);
		addDriverComparator(this.comparator);
		setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
		
		getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
		getTaskConfig().setRelativeMemoryDriver(combine_frac);
		getTaskConfig().setFilehandlesDriver(2);
		
		final GroupReduceCombineDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = 
				new GroupReduceCombineDriver<>();
		
		try {
			testDriver(testTask, MockFailingCombiningReduceStub.class);
			fail("Exception not forwarded.");
		}
		catch (ExpectedTestException etex) {
			// good!
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #30
Source Project: Flink-CEPplus   Author: ljygz   File: ReOpenableHashTableITCase.java    License: Apache License 2.0 5 votes vote down vote up
private MutableObjectIterator<Tuple2<Integer, Integer>> getProbeInput(final int numKeys,
																	  final int probeValsPerKey, final int repeatedValue1, final int repeatedValue2) {
	MutableObjectIterator<Tuple2<Integer, Integer>> probe1 = new UniformIntTupleGenerator(numKeys, probeValsPerKey, true);
	MutableObjectIterator<Tuple2<Integer, Integer>> probe2 = new TestData.ConstantIntIntTuplesIterator(repeatedValue1, 17, 5);
	MutableObjectIterator<Tuple2<Integer, Integer>> probe3 = new TestData.ConstantIntIntTuplesIterator(repeatedValue2, 23, 5);
	List<MutableObjectIterator<Tuple2<Integer, Integer>>> probes = new ArrayList<>();
	probes.add(probe1);
	probes.add(probe2);
	probes.add(probe3);
	return new UnionIterator<>(probes);
}