Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.IOManager

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.IOManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public NonReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 
Example 2
public NonReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 
Example 3
Source Project: flink   Source File: LongHybridHashTable.java    License: Apache License 2.0 6 votes vote down vote up
public LongHybridHashTable(
		Configuration conf,
		Object owner,
		BinaryRowDataSerializer buildSideSerializer,
		BinaryRowDataSerializer probeSideSerializer,
		MemoryManager memManager,
		long reservedMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount) {
	super(conf, owner, memManager, reservedMemorySize, ioManager, avgRecordLen, buildRowCount, false);
	this.buildSideSerializer = buildSideSerializer;
	this.probeSideSerializer = probeSideSerializer;

	this.partitionsBeingBuilt = new ArrayList<>();
	this.partitionsPending = new ArrayList<>();

	createPartitions(initPartitionFanOut, 0);
}
 
Example 4
Source Project: flink   Source File: AbstractBinaryExternalMerger.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractBinaryExternalMerger(
		IOManager ioManager,
		int pageSize,
		int maxFanIn,
		SpillChannelManager channelManager,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize) {
	this.ioManager = ioManager;
	this.pageSize = pageSize;
	this.maxFanIn = maxFanIn;
	this.channelManager = channelManager;
	this.compressionEnable = compressionEnable;
	this.compressionCodecFactory = compressionCodecFactory;
	this.compressionBlockSize = compressionBlockSize;
}
 
Example 5
Source Project: flink   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example 6
Source Project: flink   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 
Example 7
Source Project: Flink-CEPplus   Source File: ReOpenableHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 
Example 8
Source Project: flink   Source File: AbstractBinaryExternalMerger.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractBinaryExternalMerger(
		IOManager ioManager,
		int pageSize,
		int maxFanIn,
		SpillChannelManager channelManager,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize) {
	this.ioManager = ioManager;
	this.pageSize = pageSize;
	this.maxFanIn = maxFanIn;
	this.channelManager = channelManager;
	this.compressionEnable = compressionEnable;
	this.compressionCodecFactory = compressionCodecFactory;
	this.compressionBlockSize = compressionBlockSize;
}
 
Example 9
public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example 10
public ReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example 11
Source Project: flink   Source File: FileChannelUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 
Example 12
Source Project: flink   Source File: CompressedBlockChannelReader.java    License: Apache License 2.0 6 votes vote down vote up
public CompressedBlockChannelReader(
		IOManager ioManager,
		ID channel,
		LinkedBlockingQueue<MemorySegment> blockQueue,
		BlockCompressionFactory codecFactory,
		int preferBlockSize,
		int segmentSize) throws IOException {
	this.reader = ioManager.createBufferFileReader(channel, this);
	this.blockQueue = blockQueue;
	copyCompress = preferBlockSize > segmentSize * 2;
	int blockSize = copyCompress ? preferBlockSize : segmentSize;
	this.decompressor = codecFactory.getDecompressor();
	cause = new AtomicReference<>();

	if (copyCompress) {
		this.buf = new byte[blockSize];
		this.bufWrapper = ByteBuffer.wrap(buf);
	}

	BlockCompressor compressor = codecFactory.getCompressor();
	for (int i = 0; i < 2; i++) {
		MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(blockSize)]);
		reader.readInto(new NetworkBuffer(segment, this));
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 
Example 14
Source Project: Flink-CEPplus   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example 15
Source Project: flink   Source File: ReusingMergeOuterJoinIterator.java    License: Apache License 2.0 6 votes vote down vote up
public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example 16
Source Project: flink   Source File: TempBarrier.java    License: Apache License 2.0 5 votes vote down vote up
public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 
Example 17
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 18
Source Project: flink   Source File: MutableHashTable.java    License: Apache License 2.0 5 votes vote down vote up
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer,
		TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> comparator,
		List<MemorySegment> memorySegments,
		IOManager ioManager,
		boolean useBloomFilters)
{
	this(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator,
		memorySegments, ioManager, DEFAULT_RECORD_LEN, useBloomFilters);
}
 
Example 19
Source Project: flink   Source File: HashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This tests the case where no additional partition buffers are used at the point when spilling
 * is triggered, testing that overflow bucket buffers are taken into account when deciding which
 * partition to spill.
 */
@Test
public void testSpillingFreesOnlyOverflowSegments() {
	final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE;
	final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class);
	final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class);
	
	@SuppressWarnings("unchecked")
	final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class);
	
	try (final IOManager ioMan = new IOManagerAsync()) {
		final int pageSize = 32*1024;
		final int numSegments = 34;

		List<MemorySegment> memory = getMemory(numSegments, pageSize);

		MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>(
				serializer, serializer, buildComparator, probeComparator,
				pairComparator, memory, ioMan, 1, false);

		table.open(new ByteValueIterator(100000000), new ByteValueIterator(1));
		
		table.close();
		
		checkNoTempFilesRemain(ioMan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 20
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType,
MutableObjectIterator<T1> input1,
MutableObjectIterator<T2> input2,
TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
TypePairComparator<T1, T2> pairComparator,
MemoryManager memoryManager,
IOManager ioManager,
int numMemoryPages,
AbstractInvokable parentTask) throws Exception;
 
Example 21
Source Project: stateful-functions   Source File: KeyGroupStream.java    License: Apache License 2.0 5 votes vote down vote up
KeyGroupStream(
    TypeSerializer<T> serializer, IOManager ioManager, MemorySegmentPool memorySegmentPool) {
  this.serializer = Objects.requireNonNull(serializer);
  this.memoryPool = Objects.requireNonNull(memorySegmentPool);

  // SpillingBuffer requires at least 1 memory segment to be present at construction, otherwise it
  // fails
  // so we
  memorySegmentPool.ensureAtLeastOneSegmentPresent();
  this.target =
      new SpillingBuffer(ioManager, memorySegmentPool, memorySegmentPool.getSegmentSize());
}
 
Example 22
Source Project: flink   Source File: NonReusingMergeInnerJoinIterator.java    License: Apache License 2.0 5 votes vote down vote up
public NonReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);
}
 
Example 23
Source Project: flink   Source File: BinaryExternalSorter.java    License: Apache License 2.0 5 votes vote down vote up
public BinaryExternalSorter(
		final Object owner, MemoryManager memoryManager, long reservedMemorySize,
		IOManager ioManager, AbstractRowSerializer<BaseRow> inputSerializer,
		BinaryRowSerializer serializer, NormalizedKeyComputer normalizedKeyComputer,
		RecordComparator comparator, Configuration conf) throws IOException {
	this(owner, memoryManager, reservedMemorySize, ioManager,
			inputSerializer, serializer, normalizedKeyComputer, comparator,
			conf, AlgorithmOptions.SORT_SPILLING_THRESHOLD.defaultValue());
}
 
Example 24
public NonReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1,
			comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example 25
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType,
MutableObjectIterator<T1> input1,
MutableObjectIterator<T2> input2,
TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
TypePairComparator<T1, T2> pairComparator,
MemoryManager memoryManager,
IOManager ioManager,
int numMemoryPages,
AbstractInvokable parentTask) throws Exception;
 
Example 26
Source Project: flink   Source File: SpillingBuffer.java    License: Apache License 2.0 5 votes vote down vote up
public SpillingBuffer(IOManager ioManager, MemorySegmentSource memSource, int segmentSize) {
	super(memSource.nextSegment(), segmentSize, 0);
	
	this.fullSegments = new ArrayList<MemorySegment>(16);
	this.memorySource = memSource;
	this.ioManager = ioManager;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: MutableHashTable.java    License: Apache License 2.0 5 votes vote down vote up
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer,
		TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> comparator,
		List<MemorySegment> memorySegments,
		IOManager ioManager,
		boolean useBloomFilters)
{
	this(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator,
		memorySegments, ioManager, DEFAULT_RECORD_LEN, useBloomFilters);
}
 
Example 28
Source Project: flink   Source File: NonReusingMergeInnerJoinIterator.java    License: Apache License 2.0 5 votes vote down vote up
public NonReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);
}
 
Example 29
Source Project: flink   Source File: BinaryHashPartition.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are
 * added to
 * it. The spilling process must free at least one buffer, either in the partition's record
 * buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 *
 * @param ioAccess      The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
int spillPartition(
		IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			compressionEnable,
			compressionCodecFactory,
			compressionBlockSize,
			memorySegmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example 30
Source Project: flink   Source File: AbstractMergeInnerJoinIterator.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);
}