org.apache.flink.util.MutableObjectIterator Java Examples

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
		throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
	}

	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);

	IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
	IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();

	this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
	this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);

	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}
 

public NonReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 

private Map<Integer, Collection<String>> collectData(MutableObjectIterator<Tuple2<Integer, String>> iter)
		throws Exception {
	final Map<Integer, Collection<String>> map = new HashMap<>();
	Tuple2<Integer, String> pair = new Tuple2<>();

	while ((pair = iter.next(pair)) != null) {
		final Integer key = pair.getField(0);

		if (!map.containsKey(key)) {
			map.put(key, new ArrayList<String>());
		}

		Collection<String> values = map.get(key);
		final String value = pair.getField(1);
		values.add(value);
	}

	return map;
}
 

private int join(
		MyHashTable table,
		MutableObjectIterator<BinaryRow> buildInput,
		MutableObjectIterator<BinaryRow> probeInput) throws IOException {
	int count = 0;

	BinaryRow reuseBuildSizeRow = buildSideSerializer.createInstance();
	BinaryRow buildRow;
	while ((buildRow = buildInput.next(reuseBuildSizeRow)) != null) {
		table.putBuildRow(buildRow);
	}
	table.endBuild();

	BinaryRow probeRow = probeSideSerializer.createInstance();
	while ((probeRow = probeInput.next(probeRow)) != null) {
		if (table.tryProbe(probeRow)){
			count += joinWithNextKey(table);
		}
	}

	while (table.nextMatching()){
		count += joinWithNextKey(table);
	}
	return count;
}
 

public ReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 

public MutableObjectIterator<Tuple2<BinaryRow, BinaryRow>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRow, BinaryRow>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 

@Override
public void run() {
	final MutableObjectIterator<T> input = this.input;
	final TypeSerializer<T> serializer = this.serializer;
	final SpillingBuffer buffer = this.buffer;
	
	try {
		T record = serializer.createInstance();
		
		while (this.running && ((record = input.next(record)) != null)) {
			serializer.serialize(record, buffer);
		}
		
		TempBarrier.this.writingDone();
	}
	catch (Throwable t) {
		TempBarrier.this.setException(t);
	}
}
 

@Override
public <X> MutableObjectIterator<X> getInput(int index) {
	MutableObjectIterator<IN> in = this.inputs.get(index);
	if (in == null) {
		// waiting from sorter
		try {
			in = this.sorters.get(index).getIterator();
		} catch (InterruptedException e) {
			throw new RuntimeException("Interrupted");
		}
		this.inputs.set(index, in);
	}
	
	@SuppressWarnings("unchecked")
	MutableObjectIterator<X> input = (MutableObjectIterator<X>) this.inputs.get(index);
	return input;
}
 

public ReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

@Test
public void testSparseProbeSpilling() throws IOException, MemoryAllocationException {
	final int numBuildKeys = 1000000;
	final int numBuildVals = 1;
	final int numProbeKeys = 20;
	final int numProbeVals = 1;

	MutableObjectIterator<BinaryRow> buildInput = new UniformBinaryRowGenerator(
			numBuildKeys, numBuildVals, false);
	final MyHashTable table = new MyHashTable(100 * PAGE_SIZE);

	int expectedNumResults = (Math.min(numProbeKeys, numBuildKeys) * numBuildVals)
			* numProbeVals;

	int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true));

	Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);

	table.close();

	table.free();
}
 

@Override
public MutableObjectIterator<E> getIterator() throws InterruptedException {
	synchronized (this.iteratorLock) {
		// wait while both the iterator and the exception are not set
		while (this.iterator == null && this.iteratorException == null) {
			this.iteratorLock.wait();
		}
		
		if (this.iteratorException != null) {
			throw new RuntimeException("Error obtaining the sorted input: " + this.iteratorException.getMessage(),
				this.iteratorException);
		}
		else {
			return this.iterator;
		}
	}
}
 

@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

/**
 * Initializes the input readers of the DataSinkTask.
 * 
 * @throws RuntimeException
 *         Thrown in case of invalid task input configuration.
 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
	int numGates = 0;
	//  ---------------- create the input readers ---------------------
	// in case where a logical input unions multiple physical inputs, create a union reader
	final int groupSize = this.config.getGroupSize(0);
	numGates += groupSize;
	if (groupSize == 1) {
		// non-union case
		inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
				getEnvironment().getInputGate(0),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else if (groupSize > 1){
		// union case
		inputReader = new MutableRecordReader<IOReadableWritable>(
				new UnionInputGate(getEnvironment().getAllInputGates()),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else {
		throw new Exception("Illegal input group size in task configuration: " + groupSize);
	}
	
	this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
	@SuppressWarnings({ "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
	this.reader = (MutableObjectIterator<IT>)iter;

	// final sanity check
	if (numGates != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

public SortMergeFullOuterJoinIterator(
		BinaryRowSerializer serializer1, BinaryRowSerializer serializer2,
		Projection<BaseRow, BinaryRow> projection1,
		Projection<BaseRow, BinaryRow> projection2,
		RecordComparator keyComparator,
		MutableObjectIterator<BinaryRow> iterator1,
		MutableObjectIterator<BinaryRow> iterator2,
		ResettableExternalBuffer buffer1,
		ResettableExternalBuffer buffer2,
		boolean[] filterNulls) throws IOException {
	this.projection1 = projection1;
	this.projection2 = projection2;
	this.keyComparator = keyComparator;
	this.iterator1 = iterator1;
	this.iterator2 = iterator2;

	this.row1 = serializer1.createInstance();
	this.row2 = serializer2.createInstance();
	this.buffer1 = buffer1;
	this.buffer2 = buffer2;
	this.nullFilterKeys = NullAwareJoinHelper.getNullFilterKeys(filterNulls);
	this.nullSafe = nullFilterKeys.length == 0;
	this.filterAllNulls = nullFilterKeys.length == filterNulls.length;

	nextRow1();
	nextRow2();
}
 

@Test
public void testAllReduceDriverImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(result);
		context.setComparator1(comparator);
		context.setUdf(new ConcatSumReducer());
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

/**
 * Initializes the input readers of the DataSinkTask.
 * 
 * @throws RuntimeException
 *         Thrown in case of invalid task input configuration.
 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
	int numGates = 0;
	//  ---------------- create the input readers ---------------------
	// in case where a logical input unions multiple physical inputs, create a union reader
	final int groupSize = this.config.getGroupSize(0);
	numGates += groupSize;
	if (groupSize == 1) {
		// non-union case
		inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
				getEnvironment().getInputGate(0),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else if (groupSize > 1){
		// union case
		inputReader = new MutableRecordReader<IOReadableWritable>(
				new UnionInputGate(getEnvironment().getAllInputGates()),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else {
		throw new Exception("Illegal input group size in task configuration: " + groupSize);
	}
	
	this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
	@SuppressWarnings({ "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
	this.reader = (MutableObjectIterator<IT>)iter;

	// final sanity check
	if (numGates != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType,
MutableObjectIterator<T1> input1,
MutableObjectIterator<T2> input2,
TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
TypePairComparator<T1, T2> pairComparator,
MemoryManager memoryManager,
IOManager ioManager,
int numMemoryPages,
AbstractInvokable parentTask) throws Exception;
 

public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2,
							TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
							TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
							TypePairComparator<T1, T2> pairComparator,
							MemoryManager memoryManager,
							IOManager ioManager,
							int numMemoryPages,
							AbstractInvokable parentTask) throws MemoryAllocationException {
	if (numMemoryPages < 2) {
		throw new IllegalArgumentException("Merger needs at least 2 memory pages.");
	}

	this.pairComparator = pairComparator;
	this.serializer1 = serializer1;
	this.serializer2 = serializer2;

	this.memoryManager = memoryManager;
	this.ioManager = ioManager;

	this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate());
	this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate());

	final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1;
	this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2,
			(numMemoryPages - numPagesForSpiller), parentTask);
	this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller);
}
 

public void buildTableWithUniqueKey(final MutableObjectIterator<T> input) throws IOException {
	// go over the complete input and insert every element into the hash table
	
	T value;
	while (this.running && (value = input.next()) != null) {
		insertOrReplaceRecord(value);
	}
}
 

public IteratorWrappingTestSingleInputGate<Record> addInput(MutableObjectIterator<Record> inputIterator) {
	try {
		final IteratorWrappingTestSingleInputGate<Record> reader = new IteratorWrappingTestSingleInputGate<Record>(bufferSize, Record.class, inputIterator);

		inputs.add(reader.getInputGate());

		return reader;
	}
	catch (Throwable t) {
		throw new RuntimeException("Error setting up mock readers: " + t.getMessage(), t);
	}
}
 

@Test
public void testCombineSpillingDisableObjectReuse() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");

	MaterializedCountCombiner comb = new MaterializedCountCombiner();

	// set maxNumFileHandles = 2 to trigger multiple channel merging
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 2, 0.005f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();

	for (int i = 0; i < noKeyCnt; i++) {
		rec.setField(i, 0);
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 1);
			reader.emit(rec);
		}
	}
	reader.close();

	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();
	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(4950, result.next().intValue());
	}

	merger.close();
}
 

/**
 * Creates the initial hash table. This method sets up partitions, hash index, and inserts
 * the data from the given iterator.
 * 
 * @param input The iterator with the build side data.
 * @throws IOException Thrown, if an element could not be fetched and deserialized from
 *                     the iterator, or if serialization fails.
 */
protected void buildInitialTable(final MutableObjectIterator<BT> input) throws IOException {
	// create the partitions
	final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size());
	if (partitionFanOut > MAX_NUM_PARTITIONS) {
		throw new RuntimeException("Hash join partitions estimate exeeds maximum number of partitions."); 
	}
	createPartitions(partitionFanOut, 0);
	
	// set up the table structure. the write behind buffers are taken away, as are one buffer per partition
	final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize, 
		partitionFanOut, this.avgRecordLen);
	initTable(numBuckets, (byte) partitionFanOut);
	
	final TypeComparator<BT> buildTypeComparator = this.buildSideComparator;
	BT record = this.buildSideSerializer.createInstance();

	// go over the complete input and insert every element into the hash table
	while (this.running && ((record = input.next(record)) != null)) {
		final int hashCode = hash(buildTypeComparator.hash(record), 0);
		insertIntoTable(record, hashCode);
	}
	
	if (!this.running) {
		return;
	}

	// finalize the partitions
	for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
		HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i);
		p.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
	}
}
 

@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	do {
		generator.next(record);
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
	Tuple2<Integer, String> readTarget = new Tuple2<>();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		generator.next(record);
		
		int rk = readTarget.f0;
		int gk = record.f0;
		
		String rv = readTarget.f1;
		String gv = record.f1;
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Override
@SuppressWarnings({"unchecked", "rawtypes"})
protected <T1, T2, T3> AbstractMergeOuterJoinIterator createOuterJoinIterator(OuterJoinType outerJoinType, MutableObjectIterator<T1> input1,
																		  MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1,
																		  TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
																		  TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager,
																		  int numMemoryPages, AbstractInvokable parentTask) throws Exception {
	return new ReusingMergeOuterJoinIterator(outerJoinType, input1, input2, serializer1, comparator1,
			serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);
}
 

private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException {
	final MutableObjectIterator<X> results = hashTable.getEntryIterator();
	final Collector<X> output = this.finalOutputCollector;
	X record = solutionTypeSerializer.getSerializer().createInstance();

	while ((record = results.next(record)) != null) {
		output.collect(record);
	}
}
 

public ReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.nextBuildSideObject = serializer2.createInstance();
	this.tempBuildSideRecord = serializer2.createInstance();

	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1, comparator1, pairComparator,
		memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

@Override
public boolean callWithNextKey(FlatJoinFunction<V1, V2, O> matchFunction, Collector<O> collector)
throws Exception
{
	if (this.hashJoin.nextRecord())
	{
		// we have a next record, get the iterators to the probe and build side values
		final MutableObjectIterator<V2> buildSideIterator = this.hashJoin.getBuildSideIterator();
		final V1 probeRecord = this.hashJoin.getCurrentProbeRecord();
		V2 nextBuildSideRecord = buildSideIterator.next(this.nextBuildSideObject);

		if (probeRecord != null && nextBuildSideRecord != null) {
			matchFunction.join(probeRecord, nextBuildSideRecord, collector);

			while (this.running && ((nextBuildSideRecord = buildSideIterator.next(nextBuildSideRecord)) != null)) {
				matchFunction.join(probeRecord, nextBuildSideRecord, collector);
			}
		} else {
			if (probeSideOuterJoin && probeRecord != null && nextBuildSideRecord == null) {
				matchFunction.join(probeRecord, null, collector);
			}

			if (buildSideOuterJoin && probeRecord == null && nextBuildSideRecord != null) {
				matchFunction.join(null, nextBuildSideRecord, collector);
				while (this.running && ((nextBuildSideRecord = buildSideIterator.next(nextBuildSideRecord)) != null)) {
					matchFunction.join(null, nextBuildSideRecord, collector);
				}
			}
		}

		return true;
	}
	else {
		return false;
	}
}