Java Code Examples for org.apache.flink.runtime.operators.sort.UnilateralSortMerger

The following examples show how to use org.apache.flink.runtime.operators.sort.UnilateralSortMerger. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void addInputSorted(MutableObjectIterator<IN> input, TypeSerializer<IN> serializer, TypeComparator<IN> comp) throws Exception {
	this.inputSerializers.add(serializer);
	UnilateralSortMerger<IN> sorter = new UnilateralSortMerger<>(
			this.memManager,
			this.ioManager,
			input,
			this.owner,
			new RuntimeSerializerFactory<>(serializer, (Class<IN>) serializer.createInstance().getClass()),
			comp,
			this.perSortFractionMem,
			32,
			0.8f,
			true /*use large record handler*/,
			false
	);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());
	
	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 5
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 6
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 7
Source Project: flink   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void addInputSorted(MutableObjectIterator<IN> input, TypeSerializer<IN> serializer, TypeComparator<IN> comp) throws Exception {
	this.inputSerializers.add(serializer);
	UnilateralSortMerger<IN> sorter = new UnilateralSortMerger<>(
			this.memManager,
			this.ioManager,
			input,
			this.owner,
			new RuntimeSerializerFactory<>(serializer, (Class<IN>) serializer.createInstance().getClass()),
			comp,
			this.perSortFractionMem,
			32,
			0.8f,
			true /*use large record handler*/,
			false
	);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 8
Source Project: flink   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 9
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 10
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 11
Source Project: flink   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void addInputSorted(MutableObjectIterator<IN> input, TypeSerializer<IN> serializer, TypeComparator<IN> comp) throws Exception {
	this.inputSerializers.add(serializer);
	UnilateralSortMerger<IN> sorter = new UnilateralSortMerger<>(
			this.memManager,
			this.ioManager,
			input,
			this.owner,
			new RuntimeSerializerFactory<>(serializer, (Class<IN>) serializer.createInstance().getClass()),
			comp,
			this.perSortFractionMem,
			32,
			0.8f,
			true /*use large record handler*/,
			false
	);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 12
Source Project: flink   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: DriverTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<Record> input, RecordComparator comp) throws Exception {
	UnilateralSortMerger<Record> sorter = new UnilateralSortMerger<Record>(
			this.memManager, this.ioManager, input, this.owner, RecordSerializerFactory.get(), comp,
			this.perSortFractionMem, 32, 0.8f, true /*use large record handler*/, true);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 14
Source Project: Flink-CEPplus   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<IN> input,
							TypeSerializer<IN> serializer,
							TypeComparator<IN> comp) throws Exception
{
	this.input = null;
	this.inputSerializer = serializer;
	this.sorter = new UnilateralSortMerger<IN>(
			this.memManager, this.ioManager, input, this.owner,
			this.<IN>getInputSerializer(0),
			comp,
			this.perSortFractionMem, 32, 0.8f,
			true /*use large record handler*/,
			false);
}
 
Example 15
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<Record> input, RecordComparator comp) throws Exception {
	UnilateralSortMerger<Record> sorter = new UnilateralSortMerger<Record>(
			this.memManager, this.ioManager, input, this.owner, RecordSerializerFactory.get(), comp,
			this.perSortFractionMem, 32, 0.8f, true /*use large record handler*/, true);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 16
Source Project: flink   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<IN> input,
							TypeSerializer<IN> serializer,
							TypeComparator<IN> comp) throws Exception
{
	this.input = null;
	this.inputSerializer = serializer;
	this.sorter = new UnilateralSortMerger<IN>(
			this.memManager, this.ioManager, input, this.owner,
			this.<IN>getInputSerializer(0),
			comp,
			this.perSortFractionMem, 32, 0.8f,
			true /*use large record handler*/,
			false);
}
 
Example 17
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<Record> input, RecordComparator comp) throws Exception {
	UnilateralSortMerger<Record> sorter = new UnilateralSortMerger<Record>(
			this.memManager, this.ioManager, input, this.owner, RecordSerializerFactory.get(), comp,
			this.perSortFractionMem, 32, 0.8f, true /*use large record handler*/, true);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 
Example 18
Source Project: flink   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputSorted(MutableObjectIterator<IN> input,
							TypeSerializer<IN> serializer,
							TypeComparator<IN> comp) throws Exception
{
	this.input = null;
	this.inputSerializer = serializer;
	this.sorter = new UnilateralSortMerger<IN>(
			this.memManager, this.ioManager, input, this.owner,
			this.<IN>getInputSerializer(0),
			comp,
			this.perSortFractionMem, 32, 0.8f,
			true /*use large record handler*/,
			false);
}
 
Example 19
Source Project: Flink-CEPplus   Source File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSortBothMerge() {
	try {
		
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter1 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, 
				this.comparator1.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter2 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, 
				this.comparator2.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput1 = sorter1.getIterator();
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput2 = sorter2.getIterator();
		
		// compare with iterator values
		ReusingMergeInnerJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingMergeInnerJoinIterator<>(sortedInput1, sortedInput2,
					this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		sorter1.close();
		sorter2.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Sort-Merge Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 21
Source Project: flink   Source File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 
Example 22
Source Project: flink   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSortBothMerge() {
	try {
		
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter1 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, 
				this.comparator1.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter2 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, 
				this.comparator2.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput1 = sorter1.getIterator();
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput2 = sorter2.getIterator();
		
		// compare with iterator values
		ReusingMergeInnerJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingMergeInnerJoinIterator<>(sortedInput1, sortedInput2,
					this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		sorter1.close();
		sorter2.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Sort-Merge Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 
Example 23
Source Project: flink   Source File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 
Example 24
Source Project: flink   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSortBothMerge() {
	try {
		
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter1 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, 
				this.comparator1.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter2 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, 
				this.comparator2.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput1 = sorter1.getIterator();
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput2 = sorter2.getIterator();
		
		// compare with iterator values
		ReusingMergeInnerJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingMergeInnerJoinIterator<>(sortedInput1, sortedInput2,
					this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		sorter1.close();
		sorter2.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Sort-Merge Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}