Java Code Examples for org.apache.flink.core.memory.MemorySegment#putLong()

/**
 * Inserts the given record into the hash table.
 * Note: this method doesn't care about whether a record with the same key is already present.
 * @param record The record to insert.
 * @throws IOException (EOFException specifically, if memory ran out)
    */
@Override
public void insert(T record) throws IOException {
	if (closed) {
		return;
	}

	final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record));
	final int bucket = hashCode & numBucketsMask;
	final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket
	final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex];
	final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment
	final long firstPointer = bucketSegment.getLong(bucketOffset);

	try {
		final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record);
		bucketSegment.putLong(bucketOffset, newFirstPointer);
	} catch (EOFException ex) {
		compactOrThrow();
		insert(record);
		return;
	}

	numElements++;
	resizeTableIfNecessary();
}
 

/**
 * Inserts the given record into the hash table.
 * Note: this method doesn't care about whether a record with the same key is already present.
 * @param record The record to insert.
 * @throws IOException (EOFException specifically, if memory ran out)
    */
@Override
public void insert(T record) throws IOException {
	if (closed) {
		return;
	}

	final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record));
	final int bucket = hashCode & numBucketsMask;
	final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket
	final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex];
	final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment
	final long firstPointer = bucketSegment.getLong(bucketOffset);

	try {
		final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record);
		bucketSegment.putLong(bucketOffset, newFirstPointer);
	} catch (EOFException ex) {
		compactOrThrow();
		insert(record);
		return;
	}

	numElements++;
	resizeTableIfNecessary();
}
 

private static void writeBytesToFixLenPart(
	MemorySegment segment, int fieldOffset, byte[] bytes, int len) {
	long firstByte = len | 0x80; // first bit is 1, other bits is len
	long sevenBytes = 0L; // real data
	if (BinaryRowData.LITTLE_ENDIAN) {
		for (int i = 0; i < len; i++) {
			sevenBytes |= ((0x00000000000000FFL & bytes[i]) << (i * 8L));
		}
	} else {
		for (int i = 0; i < len; i++) {
			sevenBytes |= ((0x00000000000000FFL & bytes[i]) << ((6 - i) * 8L));
		}
	}

	final long offsetAndSize = (firstByte << 56) | sevenBytes;

	segment.putLong(fieldOffset, offsetAndSize);
}
 

/** Same as above, but the number of bucket segments of the new table can be specified. */
private void rebuild(long newNumBucketSegments) throws IOException {
	// Get new bucket segments
	releaseBucketSegments();
	allocateBucketSegments((int)newNumBucketSegments);

	T record = buildSideSerializer.createInstance();
	try {
		EntryIterator iter = getEntryIterator();
		recordArea.resetAppendPosition();
		recordArea.setWritePosition(0);
		while ((record = iter.next(record)) != null && !closed) {
			final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record));
			final int bucket = hashCode & numBucketsMask;
			final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket
			final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex];
			final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment
			final long firstPointer = bucketSegment.getLong(bucketOffset);

			long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record);
			bucketSegment.putLong(bucketOffset, ptrToAppended);
		}
		recordArea.freeSegmentsAfterAppendPosition();
		holes = 0;

	} catch (EOFException ex) {
		throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " +
			"because we aren't allocating any new memory.");
	}
}
 

/** Same as above, but the number of bucket segments of the new table can be specified. */
private void rebuild(long newNumBucketSegments) throws IOException {
	// Get new bucket segments
	releaseBucketSegments();
	allocateBucketSegments((int)newNumBucketSegments);

	T record = buildSideSerializer.createInstance();
	try {
		EntryIterator iter = getEntryIterator();
		recordArea.resetAppendPosition();
		recordArea.setWritePosition(0);
		while ((record = iter.next(record)) != null && !closed) {
			final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record));
			final int bucket = hashCode & numBucketsMask;
			final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket
			final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex];
			final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment
			final long firstPointer = bucketSegment.getLong(bucketOffset);

			long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record);
			bucketSegment.putLong(bucketOffset, ptrToAppended);
		}
		recordArea.freeSegmentsAfterAppendPosition();
		holes = 0;

	} catch (EOFException ex) {
		throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " +
			"because we aren't allocating any new memory.");
	}
}
 

protected void initTable(int numBuckets, byte numPartitions) {
	final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
	final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1);
	final MemorySegment[] table = new MemorySegment[numSegs];
	
	ensureNumBuffersReturned(numSegs);
	
	// go over all segments that are part of the table
	for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
		final MemorySegment seg = getNextBuffer();
		
		// go over all buckets in the segment
		for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
			final int bucketOffset = k * HASH_BUCKET_SIZE;	
			
			// compute the partition that the bucket corresponds to
			final byte partition = assignPartition(bucket, numPartitions);
			
			// initialize the header fields
			seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
			seg.put(bucketOffset + HEADER_STATUS_OFFSET, BUCKET_STATUS_IN_MEMORY);
			seg.putShort(bucketOffset + HEADER_COUNT_OFFSET, (short) 0);
			seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
			seg.putShort(bucketOffset + HEADER_PROBED_FLAGS_OFFSET, (short) 0);
		}
		
		table[i] = seg;
	}
	this.buckets = table;
	this.numBuckets = numBuckets;
	
	if (useBloomFilters) {
		initBloomFilter(numBuckets);
	}
}
 

private void initTable(int numBuckets, byte numPartitions) {
	final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
	final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1);
	final MemorySegment[] table = new MemorySegment[numSegs];
	
	// go over all segments that are part of the table
	for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
		final MemorySegment seg = getNextBuffer();
		
		// go over all buckets in the segment
		for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
			final int bucketOffset = k * HASH_BUCKET_SIZE;	
			
			// compute the partition that the bucket corresponds to
			final byte partition = assignPartition(bucket, numPartitions);
			
			// initialize the header fields
			seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
			seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
			seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
		}
		
		table[i] = seg;
	}
	this.buckets = table;
	this.numBuckets = numBuckets;
}
 

private void initMemorySegment(MemorySegment seg) {
	// go over all buckets in the segment
	for (int k = 0; k < table.bucketsPerSegment; k++) {
		final int bucketOffset = k * BUCKET_SIZE;

		// init count and probeFlag and forward pointer together.
		seg.putLong(bucketOffset + HEADER_COUNT_OFFSET, BUCKET_HEADER_INIT);
	}
}
 

protected void initTable(int numBuckets, byte numPartitions) {
	final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
	final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1);
	final MemorySegment[] table = new MemorySegment[numSegs];
	
	ensureNumBuffersReturned(numSegs);
	
	// go over all segments that are part of the table
	for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
		final MemorySegment seg = getNextBuffer();
		
		// go over all buckets in the segment
		for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
			final int bucketOffset = k * HASH_BUCKET_SIZE;	
			
			// compute the partition that the bucket corresponds to
			final byte partition = assignPartition(bucket, numPartitions);
			
			// initialize the header fields
			seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
			seg.put(bucketOffset + HEADER_STATUS_OFFSET, BUCKET_STATUS_IN_MEMORY);
			seg.putShort(bucketOffset + HEADER_COUNT_OFFSET, (short) 0);
			seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
			seg.putShort(bucketOffset + HEADER_PROBED_FLAGS_OFFSET, (short) 0);
		}
		
		table[i] = seg;
	}
	this.buckets = table;
	this.numBuckets = numBuckets;
	
	if (useBloomFilters) {
		initBloomFilter(numBuckets);
	}
}
 

@Override
public void swapKey(MemorySegment segI, int offsetI,
		MemorySegment segJ, int offsetJ) {
	long temp0 = segI.getLong(offsetI);
	segI.putLong(offsetI, segJ.getLong(offsetJ));
	segJ.putLong(offsetJ, temp0);
}
 

@Test
public void testBucketsNotFulfillSegment() throws Exception {
	final int numKeys = 10000;
	final int buildValsPerKey = 3;
	final int probeValsPerKey = 10;

	// create a build input that gives 30000 pairs with 3 values sharing the same key
	MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);

	// create a probe input that gives 100000 pairs with 10 values sharing a key
	MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);

	// allocate the memory for the HashTable
	MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(35 * PAGE_SIZE).build();
	// ----------------------------------------------------------------------------------------

	final BinaryHashTable table = new BinaryHashTable(conf, new Object(),
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(),
			memManager, 35 * PAGE_SIZE, ioManager, 24, 200000,
			true, HashJoinType.INNER, null, false, new boolean[]{true}, false);

	// For FLINK-2545, the buckets data may not fulfill it's buffer, for example, the buffer may contains 256 buckets,
	// while hash table only assign 250 bucket on it. The unused buffer bytes may contains arbitrary data, which may
	// influence hash table if forget to skip it. To mock this, put the invalid bucket data(partition=1, inMemory=true, count=-1)
	// at the end of buffer.
	int totalPages = table.getInternalPool().freePages();
	for (int i = 0; i < totalPages; i++) {
		MemorySegment segment = table.getInternalPool().nextSegment();
		int newBucketOffset = segment.size() - 128;
		// initialize the header fields
		segment.put(newBucketOffset, (byte) 0);
		segment.put(newBucketOffset + 1, (byte) 0);
		segment.putShort(newBucketOffset + 2, (short) -1);
		segment.putLong(newBucketOffset + 4, ~0x0L);
		table.returnPage(segment);
	}

	int numRecordsInJoinResult = join(table, buildInput, probeInput);

	Assert.assertEquals("Wrong number of records in join result.", numKeys * buildValsPerKey * probeValsPerKey, numRecordsInJoinResult);

	table.close();
	table.free();
}
 

/**
 * Update the address in array for given key.
 */
private void updateIndex(
		long key,
		int hashCode,
		long address,
		int size,
		MemorySegment dataSegment,
		int currentPositionInSegment) throws IOException {
	assert (numKeys <= numBuckets / 2);
	int bucketId = hashCode & numBucketsMask;

	// each bucket occupied 16 bytes (long key + long pointer to data address)
	int bucketOffset = bucketId * SPARSE_BUCKET_ELEMENT_SIZE_IN_BYTES;
	MemorySegment segment = buckets[bucketOffset >>> segmentSizeBits];
	int segOffset = bucketOffset & segmentSizeMask;
	long currAddress;

	while (true) {
		currAddress = segment.getLong(segOffset + 8);
		if (segment.getLong(segOffset) != key && currAddress != INVALID_ADDRESS) {
			// hash conflicts, the bucket is occupied by another key

			// TODO test Conflict resolution:
			// now:    +1 +1 +1... cache friendly but more conflict, so we set factor to 0.5
			// other1: +1 +2 +3... less conflict, factor can be 0.75
			// other2: Secondary hashCode... less and less conflict, but need compute hash again
			bucketId = (bucketId + 1) & numBucketsMask;
			if (segOffset + SPARSE_BUCKET_ELEMENT_SIZE_IN_BYTES < segmentSize) {
				// if the new bucket still in current segment, we only need to update offset
				// within this segment
				segOffset += SPARSE_BUCKET_ELEMENT_SIZE_IN_BYTES;
			} else {
				// otherwise, we should re-calculate segment and offset
				bucketOffset = bucketId * 16;
				segment = buckets[bucketOffset >>> segmentSizeBits];
				segOffset = bucketOffset & segmentSizeMask;
			}
		} else {
			break;
		}
	}
	if (currAddress == INVALID_ADDRESS) {
		// this is the first value for this key, put the address in array.
		segment.putLong(segOffset, key);
		segment.putLong(segOffset + 8, address);
		numKeys += 1;
		// dataSegment may be null if we only have to rehash bucket area
		if (dataSegment != null) {
			dataSegment.putLong(currentPositionInSegment, toAddrAndLen(INVALID_ADDRESS, size));
		}
		if (numKeys * 2 > numBuckets) {
			resize();
		}
	} else {
		// there are some values for this key, put the address in the front of them.
		dataSegment.putLong(currentPositionInSegment, toAddrAndLen(currAddress, size));
		segment.putLong(segOffset + 8, address);
	}
}
 

@Test
public void testBucketsNotFulfillSegment() throws Exception {
	final int NUM_KEYS = 10000;
	final int BUILD_VALS_PER_KEY = 3;
	final int PROBE_VALS_PER_KEY = 10;

	// create a build input that gives 30000 pairs with 3 values sharing the same key
	MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

	// create a probe input that gives 100000 pairs with 10 values sharing a key
	MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

	// allocate the memory for the HashTable
	List<MemorySegment> memSegments;
	try {
		// 33 is minimum number of pages required to perform hash join this inputs
		memSegments = this.memManager.allocatePages(MEM_OWNER, 33);
	}
	catch (MemoryAllocationException maex) {
		fail("Memory for the Join could not be provided.");
		return;
	}

	// For FLINK-2545, the buckets data may not fulfill it's buffer, for example, the buffer may contains 256 buckets,
	// while hash table only assign 250 bucket on it. The unused buffer bytes may contains arbitrary data, which may
	// influence hash table if forget to skip it. To mock this, put the invalid bucket data(partition=1, inMemory=true, count=-1)
	// at the end of buffer.
	for (MemorySegment segment : memSegments) {
		int newBucketOffset = segment.size() - 128;
		// initialize the header fields
		segment.put(newBucketOffset + 0, (byte)0);
		segment.put(newBucketOffset + 1, (byte)0);
		segment.putShort(newBucketOffset + 2, (short) -1);
		segment.putLong(newBucketOffset + 4, ~0x0L);
	}

	// ----------------------------------------------------------------------------------------

	final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
		this.pairBuildSideAccesssor, this.pairProbeSideAccesssor,
		this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
		memSegments, ioManager);
	join.open(buildInput, probeInput);

	final IntPair recordReuse = new IntPair();
	int numRecordsInJoinResult = 0;

	while (join.nextRecord()) {
		MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
		while (buildSide.next(recordReuse) != null) {
			numRecordsInJoinResult++;
		}
	}
	Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);

	join.close();
	this.memManager.release(join.getFreedMemory());
}
 

@Test
public void testBucketsNotFulfillSegment() throws Exception {
	final int numKeys = 10000;
	final int buildValsPerKey = 3;
	final int probeValsPerKey = 10;

	// create a build input that gives 30000 pairs with 3 values sharing the same key
	MutableObjectIterator<BinaryRow> buildInput = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);

	// create a probe input that gives 100000 pairs with 10 values sharing a key
	MutableObjectIterator<BinaryRow> probeInput = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);

	// allocate the memory for the HashTable
	MemoryManager memManager = new MemoryManager(35 * PAGE_SIZE, 1);
	// ----------------------------------------------------------------------------------------

	final BinaryHashTable table = new BinaryHashTable(conf, new Object(),
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(),
			memManager, 35 * PAGE_SIZE, ioManager, 24, 200000,
			true, HashJoinType.INNER, null, false, new boolean[]{true}, false);

	// For FLINK-2545, the buckets data may not fulfill it's buffer, for example, the buffer may contains 256 buckets,
	// while hash table only assign 250 bucket on it. The unused buffer bytes may contains arbitrary data, which may
	// influence hash table if forget to skip it. To mock this, put the invalid bucket data(partition=1, inMemory=true, count=-1)
	// at the end of buffer.
	for (MemorySegment segment : table.getFreedMemory()) {
		int newBucketOffset = segment.size() - 128;
		// initialize the header fields
		segment.put(newBucketOffset, (byte) 0);
		segment.put(newBucketOffset + 1, (byte) 0);
		segment.putShort(newBucketOffset + 2, (short) -1);
		segment.putLong(newBucketOffset + 4, ~0x0L);
	}

	int numRecordsInJoinResult = join(table, buildInput, probeInput);

	Assert.assertEquals("Wrong number of records in join result.", numKeys * buildValsPerKey * probeValsPerKey, numRecordsInJoinResult);

	table.close();
	table.free();
}
 

@Test
public void testBucketsNotFulfillSegment() throws Exception {
	final int NUM_KEYS = 10000;
	final int BUILD_VALS_PER_KEY = 3;
	final int PROBE_VALS_PER_KEY = 10;

	// create a build input that gives 30000 pairs with 3 values sharing the same key
	MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);

	// create a probe input that gives 100000 pairs with 10 values sharing a key
	MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);

	// allocate the memory for the HashTable
	List<MemorySegment> memSegments;
	try {
		// 33 is minimum number of pages required to perform hash join this inputs
		memSegments = this.memManager.allocatePages(MEM_OWNER, 33);
	}
	catch (MemoryAllocationException maex) {
		fail("Memory for the Join could not be provided.");
		return;
	}

	// For FLINK-2545, the buckets data may not fulfill it's buffer, for example, the buffer may contains 256 buckets,
	// while hash table only assign 250 bucket on it. The unused buffer bytes may contains arbitrary data, which may
	// influence hash table if forget to skip it. To mock this, put the invalid bucket data(partition=1, inMemory=true, count=-1)
	// at the end of buffer.
	for (MemorySegment segment : memSegments) {
		int newBucketOffset = segment.size() - 128;
		// initialize the header fields
		segment.put(newBucketOffset + 0, (byte)0);
		segment.put(newBucketOffset + 1, (byte)0);
		segment.putShort(newBucketOffset + 2, (short) -1);
		segment.putLong(newBucketOffset + 4, ~0x0L);
	}

	// ----------------------------------------------------------------------------------------

	final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(
		this.pairBuildSideAccesssor, this.pairProbeSideAccesssor,
		this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator,
		memSegments, ioManager);
	join.open(buildInput, probeInput);

	final IntPair recordReuse = new IntPair();
	int numRecordsInJoinResult = 0;

	while (join.nextRecord()) {
		MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
		while (buildSide.next(recordReuse) != null) {
			numRecordsInJoinResult++;
		}
	}
	Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);

	join.close();
	this.memManager.release(join.getFreedMemory());
}
 

/**
 * Compacts (garbage collects) partition with copy-compact strategy using compaction partition
 * 
 * @param partitionNumber partition to compact
 * @throws IOException 
 */
private void compactPartition(final int partitionNumber) throws IOException {
	// do nothing if table was closed, parameter is invalid or no garbage exists
	if (this.closed || partitionNumber >= this.partitions.size() || this.partitions.get(partitionNumber).isCompacted()) {
		return;
	}
	// release all segments owned by compaction partition
	this.compactionMemory.clearAllMemory(availableMemory);
	this.compactionMemory.allocateSegments(1);
	this.compactionMemory.pushDownPages();
	T tempHolder = this.buildSideSerializer.createInstance();
	final int numPartitions = this.partitions.size();
	InMemoryPartition<T> partition = this.partitions.remove(partitionNumber);
	MemorySegment[] overflowSegments = partition.overflowSegments;
	long pointer;
	int pointerOffset;
	int bucketOffset;
	final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
	for (int i = 0, bucket = partitionNumber; i < this.buckets.length && bucket < this.numBuckets; i++) {
		MemorySegment segment = this.buckets[i];
		// go over all buckets in the segment belonging to the partition
		for (int k = bucket % bucketsPerSegment; k < bucketsPerSegment && bucket < this.numBuckets; k += numPartitions, bucket += numPartitions) {
			bucketOffset = k * HASH_BUCKET_SIZE;
			if((int)segment.get(bucketOffset + HEADER_PARTITION_OFFSET) != partitionNumber) {
				throw new IOException("Accessed wrong bucket! wanted: " + partitionNumber + " got: " + segment.get(bucketOffset + HEADER_PARTITION_OFFSET));
			}
			// loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
			int countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
			int numInSegment = 0;
			pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
			while (true) {
				while (numInSegment < countInSegment) {
					pointer = segment.getLong(pointerOffset);
					tempHolder = partition.readRecordAt(pointer, tempHolder);
					pointer = this.compactionMemory.appendRecord(tempHolder);
					segment.putLong(pointerOffset, pointer);
					pointerOffset += POINTER_LEN;
					numInSegment++;
				}
				// this segment is done. check if there is another chained bucket
				final long forwardPointer = segment.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
				if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
					break;
				}
				final int overflowSegNum = (int) (forwardPointer >>> 32);
				segment = overflowSegments[overflowSegNum];
				bucketOffset = (int) forwardPointer;
				countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
				pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
				numInSegment = 0;
			}
			segment = this.buckets[i];
		}
	}
	// swap partition with compaction partition
	this.compactionMemory.setPartitionNumber(partitionNumber);
	this.partitions.add(partitionNumber, compactionMemory);
	this.partitions.get(partitionNumber).overflowSegments = partition.overflowSegments;
	this.partitions.get(partitionNumber).numOverflowSegments = partition.numOverflowSegments;
	this.partitions.get(partitionNumber).nextOverflowBucket = partition.nextOverflowBucket;
	this.partitions.get(partitionNumber).setIsCompacted(true);
	//this.partitions.get(partitionNumber).pushDownPages();
	this.compactionMemory = partition;
	this.compactionMemory.resetRecordCounter();
	this.compactionMemory.setPartitionNumber(-1);
	this.compactionMemory.overflowSegments = null;
	this.compactionMemory.numOverflowSegments = 0;
	this.compactionMemory.nextOverflowBucket = 0;
	// try to allocate maximum segment count
	this.compactionMemory.clearAllMemory(this.availableMemory);
	int maxSegmentNumber = this.getMaxPartition();
	this.compactionMemory.allocateSegments(maxSegmentNumber);
	this.compactionMemory.resetRWViews();
	this.compactionMemory.pushDownPages();
}
 

/**
 * Puts previous key pointer on the given index level to key space.
 *
 * @param memorySegment memory segment for key space.
 * @param offset offset of key space in the memory segment.
 * @param totalLevel top level of the key.
 * @param level level of index.
 * @param prevKeyPointer previous key pointer on the given level.
 */
public static void putPrevIndexNode(
	MemorySegment memorySegment, int offset, int totalLevel, int level, long prevKeyPointer) {
	int of = getIndexOffset(offset, totalLevel, level);
	memorySegment.putLong(of, prevKeyPointer);
}
 

/**
 * Puts next key pointer on the given index level to key space.
 *
 * @param memorySegment memory segment for key space.
 * @param offset offset of key space in the memory segment.
 * @param level level of index.
 * @param nextKeyPointer next key pointer on the given level.
 */
public static void putNextIndexNode(MemorySegment memorySegment, int offset, int level, long nextKeyPointer) {
	memorySegment.putLong(offset + INDEX_NEXT_OFFSET_BY_LEVEL_ARRAY[level], nextKeyPointer);
}
 

/**
 * Puts the pointer of key space.
 *
 * @param memorySegment memory segment for value space.
 * @param offset offset of value space in memory segment.
 * @param keyPointer pointer to key space.
 */
public static void putKeyPointer(MemorySegment memorySegment, int offset, long keyPointer) {
	memorySegment.putLong(offset + KEY_POINTER_OFFSET, keyPointer);
}
 

/**
 * Puts the value pointer to key space.
 *
 * @param memorySegment memory segment for key space.
 * @param offset offset of key space in the memory segment.
 * @param valuePointer the value pointer.
 */
public static void putValuePointer(MemorySegment memorySegment, int offset, long valuePointer) {
	memorySegment.putLong(offset + VALUE_POINTER_OFFSET, valuePointer);
}