Java Code Examples for org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector

The following examples show how to use org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: AllReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllReduceDriver<Tuple2<String, Integer>> driver = new AllReduceDriver<Tuple2<String,Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 5
Source Project: Flink-CEPplus   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 6
Source Project: flink   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 7
Source Project: flink   Source File: AllReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllReduceDriver<Tuple2<String, Integer>> driver = new AllReduceDriver<Tuple2<String,Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 8
Source Project: flink   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 9
Source Project: flink   Source File: AbstractOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 10
Source Project: flink   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 
Example 11
Source Project: Flink-CEPplus   Source File: CombineTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingCombineTask() {
	try {
		int keyCnt = 100;
		int valCnt = 20;
		
		setInput(new UniformIntTupleGenerator(keyCnt, valCnt, false), serializer);
		addDriverComparator(this.comparator);
		addDriverComparator(this.comparator);
		setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
		
		getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
		getTaskConfig().setRelativeMemoryDriver(combine_frac);
		getTaskConfig().setFilehandlesDriver(2);
		
		final GroupReduceCombineDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = 
				new GroupReduceCombineDriver<>();
		
		try {
			testDriver(testTask, MockFailingCombiningReduceStub.class);
			fail("Exception not forwarded.");
		}
		catch (ExpectedTestException etex) {
			// good!
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelLeftOuterJoinTaskWhileBuilding() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);
	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelLeftOuterJoinTaskWhileProbing() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
	addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 14
Source Project: Flink-CEPplus   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelRightOuterJoinTaskWhileBuilding() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
	addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelRightOuterJoinTaskWhileProbing() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);
	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildFirst() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		final ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildFirstHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
						this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
						this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build First Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildSecond() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildSecondHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
					this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build Second took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 18
Source Project: flink   Source File: CombineTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingCombineTask() {
	try {
		int keyCnt = 100;
		int valCnt = 20;
		
		setInput(new UniformIntTupleGenerator(keyCnt, valCnt, false), serializer);
		addDriverComparator(this.comparator);
		addDriverComparator(this.comparator);
		setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
		
		getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
		getTaskConfig().setRelativeMemoryDriver(combine_frac);
		getTaskConfig().setFilehandlesDriver(2);
		
		final GroupReduceCombineDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = 
				new GroupReduceCombineDriver<>();
		
		try {
			testDriver(testTask, MockFailingCombiningReduceStub.class);
			fail("Exception not forwarded.");
		}
		catch (ExpectedTestException etex) {
			// good!
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 19
Source Project: flink   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelLeftOuterJoinTaskWhileBuilding() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);
	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 20
Source Project: flink   Source File: LeftOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelLeftOuterJoinTaskWhileProbing() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
	addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 21
Source Project: flink   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelRightOuterJoinTaskWhileBuilding() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
	addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 22
Source Project: flink   Source File: RightOuterJoinTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelRightOuterJoinTaskWhileProbing() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);
	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
 
Example 23
Source Project: flink   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildFirst() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		final ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildFirstHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
						this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
						this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build First Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 24
Source Project: flink   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildSecond() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildSecondHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
					this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build Second took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 25
@Test
public void testBuildFirst() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(
				collectTupleData(input1),
				collectTupleData(input2));
		
		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		// reset the generators
		generator1.reset();
		generator2.reset();
		input1.reset();
		input2.reset();

		// compare with iterator values
		ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildFirstHashJoinIterator<>(
					input1, input2, this.recordSerializer, this.record1Comparator, 
					this.recordSerializer, this.record2Comparator, this.recordPairComparator,
					this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 26
@Test
public void testBuildFirstWithHighNumberOfCommonKeys()
{
	// the size of the left and right inputs
	final int INPUT_1_SIZE = 200;
	final int INPUT_2_SIZE = 100;
	
	final int INPUT_1_DUPLICATES = 10;
	final int INPUT_2_DUPLICATES = 2000;
	final int DUPLICATE_KEY = 13;
	
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator gen1Iter = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator gen2Iter = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final TestData.TupleConstantValueIterator const1Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "LEFT String for Duplicate Keys", INPUT_1_DUPLICATES);
		final TestData.TupleConstantValueIterator const2Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "RIGHT String for Duplicate Keys", INPUT_2_DUPLICATES);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList1 = new ArrayList<>();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList2 = new ArrayList<>();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);
		
		MutableObjectIterator<Tuple2<Integer, String>> input1 = new UnionIterator<>(inList1);
		MutableObjectIterator<Tuple2<Integer, String>> input2 = new UnionIterator<>(inList2);
		
		
		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(
				collectTupleData(input1),
				collectTupleData(input2));
		
		// re-create the whole thing for actual processing
		
		// reset the generators and iterators
		generator1.reset();
		generator2.reset();
		const1Iter.reset();
		const2Iter.reset();
		gen1Iter.reset();
		gen2Iter.reset();
		
		inList1.clear();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		inList2.clear();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);

		input1 = new UnionIterator<>(inList1);
		input2 = new UnionIterator<>(inList2);
		
		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildFirstHashJoinIterator<>(
					input1, input2, this.recordSerializer, this.record1Comparator, 
					this.recordSerializer, this.record2Comparator, this.recordPairComparator,
					this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);

		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 27
@Test
public void testBuildSecond() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(
				collectTupleData(input1),
				collectTupleData(input2));
		
		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		// reset the generators
		generator1.reset();
		generator2.reset();
		input1.reset();
		input2.reset();

		// compare with iterator values			
		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>,Tuple2<Integer, String> ,Tuple2<Integer, String> > iterator =
			new ReusingBuildSecondHashJoinIterator<>(
				input1, input2, this.recordSerializer, this.record1Comparator, 
				this.recordSerializer, this.record2Comparator, this.recordPairComparator,
				this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);

		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 28
@Test
public void testBuildSecondWithHighNumberOfCommonKeys()
{
	// the size of the left and right inputs
	final int INPUT_1_SIZE = 200;
	final int INPUT_2_SIZE = 100;
	
	final int INPUT_1_DUPLICATES = 10;
	final int INPUT_2_DUPLICATES = 2000;
	final int DUPLICATE_KEY = 13;
	
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator gen1Iter = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator gen2Iter = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final TestData.TupleConstantValueIterator const1Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "LEFT String for Duplicate Keys", INPUT_1_DUPLICATES);
		final TestData.TupleConstantValueIterator const2Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "RIGHT String for Duplicate Keys", INPUT_2_DUPLICATES);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList1 = new ArrayList<>();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList2 = new ArrayList<>();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);
		
		MutableObjectIterator<Tuple2<Integer, String>> input1 = new UnionIterator<>(inList1);
		MutableObjectIterator<Tuple2<Integer, String>> input2 = new UnionIterator<>(inList2);
		
		
		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(
				collectTupleData(input1),
				collectTupleData(input2));
		
		// re-create the whole thing for actual processing
		
		// reset the generators and iterators
		generator1.reset();
		generator2.reset();
		const1Iter.reset();
		const2Iter.reset();
		gen1Iter.reset();
		gen2Iter.reset();
		
		inList1.clear();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		inList2.clear();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);

		input1 = new UnionIterator<>(inList1);
		input2 = new UnionIterator<>(inList2);
		
		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingBuildSecondHashJoinIterator<>(
				input1, input2, this.recordSerializer, this.record1Comparator, 
				this.recordSerializer, this.record2Comparator, this.recordPairComparator,
				this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 29
@Test
public void testBuildFirstAndProbeSideOuterJoin() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 1000, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);

		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = rightOuterJoinTuples(
				collectTupleData(input1),
				collectTupleData(input2));

		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		// reset the generators
		generator1.reset();
		generator2.reset();
		input1.reset();
		input2.reset();

		// compare with iterator values
		ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildFirstHashJoinIterator<>(
						input1, input2, this.recordSerializer, this.record1Comparator,
						this.recordSerializer, this.record2Comparator, this.recordPairComparator,
						this.memoryManager, ioManager, this.parentTask, 1.0, true, false, false);

		iterator.open();

		while (iterator.callWithNextKey(matcher, collector));

		iterator.close();
	
		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 30
@Test
public void testBuildFirstAndBuildSideOuterJoin() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 1000, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);

		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = leftOuterJoinTuples(
			collectTupleData(input1),
			collectTupleData(input2));

		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		// reset the generators
		generator1.reset();
		generator2.reset();
		input1.reset();
		input2.reset();

		// compare with iterator values
		ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingBuildFirstHashJoinIterator<>(
				input1, input2, this.recordSerializer, this.record1Comparator,
				this.recordSerializer, this.record2Comparator, this.recordPairComparator,
				this.memoryManager, ioManager, this.parentTask, 1.0, false, true, false);

		iterator.open();

		while (iterator.callWithNextKey(matcher, collector));

		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}