org.apache.flink.runtime.io.disk.iomanager.IOManager Java Examples
The following examples show how to use
org.apache.flink.runtime.io.disk.iomanager.IOManager.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NonReusingBuildSecondReOpenableHashJoinIterator.java From flink with Apache License 2.0 | 6 votes |
public NonReusingBuildSecondReOpenableHashJoinIterator( MutableObjectIterator<V1> firstInput, MutableObjectIterator<V2> secondInput, TypeSerializer<V1> serializer1, TypeComparator<V1> comparator1, TypeSerializer<V2> serializer2, TypeComparator<V2> comparator2, TypePairComparator<V1, V2> pairComparator, MemoryManager memManager, IOManager ioManager, AbstractInvokable ownerTask, double memoryFraction, boolean probeSideOuterJoin, boolean buildSideOuterJoin, boolean useBitmapFilters) throws MemoryAllocationException { super(firstInput, secondInput, serializer1, comparator1, serializer2, comparator2, pairComparator, memManager, ioManager, ownerTask, memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters); reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin; }
Example #2
Source File: NonReusingBuildSecondReOpenableHashJoinIterator.java From flink with Apache License 2.0 | 6 votes |
public NonReusingBuildSecondReOpenableHashJoinIterator( MutableObjectIterator<V1> firstInput, MutableObjectIterator<V2> secondInput, TypeSerializer<V1> serializer1, TypeComparator<V1> comparator1, TypeSerializer<V2> serializer2, TypeComparator<V2> comparator2, TypePairComparator<V1, V2> pairComparator, MemoryManager memManager, IOManager ioManager, AbstractInvokable ownerTask, double memoryFraction, boolean probeSideOuterJoin, boolean buildSideOuterJoin, boolean useBitmapFilters) throws MemoryAllocationException { super(firstInput, secondInput, serializer1, comparator1, serializer2, comparator2, pairComparator, memManager, ioManager, ownerTask, memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters); reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin; }
Example #3
Source File: LongHybridHashTable.java From flink with Apache License 2.0 | 6 votes |
public LongHybridHashTable( Configuration conf, Object owner, BinaryRowDataSerializer buildSideSerializer, BinaryRowDataSerializer probeSideSerializer, MemoryManager memManager, long reservedMemorySize, IOManager ioManager, int avgRecordLen, long buildRowCount) { super(conf, owner, memManager, reservedMemorySize, ioManager, avgRecordLen, buildRowCount, false); this.buildSideSerializer = buildSideSerializer; this.probeSideSerializer = probeSideSerializer; this.partitionsBeingBuilt = new ArrayList<>(); this.partitionsPending = new ArrayList<>(); createPartitions(initPartitionFanOut, 0); }
Example #4
Source File: AbstractBinaryExternalMerger.java From flink with Apache License 2.0 | 6 votes |
public AbstractBinaryExternalMerger( IOManager ioManager, int pageSize, int maxFanIn, SpillChannelManager channelManager, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize) { this.ioManager = ioManager; this.pageSize = pageSize; this.maxFanIn = maxFanIn; this.channelManager = channelManager; this.compressionEnable = compressionEnable; this.compressionCodecFactory = compressionCodecFactory; this.compressionBlockSize = compressionBlockSize; }
Example #5
Source File: UnilateralSortMerger.java From flink with Apache License 2.0 | 6 votes |
/** * Creates the spilling thread. * * @param exceptionHandler The exception handler to call for all exceptions. * @param queues The queues used to pass buffers between the threads. * @param parentTask The task that started this thread. If non-null, it is used to register this thread. * @param memManager The memory manager used to allocate buffers for the readers and writers. * @param ioManager The I/I manager used to instantiate readers and writers from. * @param serializer * @param comparator * @param sortReadMemory * @param writeMemory * @param maxNumFileHandles */ public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues, AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, TypeSerializer<E> serializer, TypeComparator<E> comparator, List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles) { super(exceptionHandler, "SortMerger spilling thread", queues, parentTask); this.memManager = memManager; this.ioManager = ioManager; this.serializer = serializer; this.comparator = comparator; this.mergeReadMemory = sortReadMemory; this.writeMemory = writeMemory; this.maxFanIn = maxNumFileHandles; this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1; }
Example #6
Source File: UnilateralSortMerger.java From flink with Apache License 2.0 | 6 votes |
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory, IOManager ioManager, MutableObjectIterator<E> input, AbstractInvokable parentTask, TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator, int numSortBuffers, int maxNumFileHandles, float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords, boolean objectReuseEnabled) throws IOException { this ( memoryManager, memory, ioManager, input, parentTask, serializerFactory, comparator, numSortBuffers, maxNumFileHandles, startSpillingFraction, noSpillingMemory, handleLargeRecords, objectReuseEnabled, new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING)); }
Example #7
Source File: ReOpenableHashPartition.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
Example #8
Source File: AbstractBinaryExternalMerger.java From flink with Apache License 2.0 | 6 votes |
public AbstractBinaryExternalMerger( IOManager ioManager, int pageSize, int maxFanIn, SpillChannelManager channelManager, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize) { this.ioManager = ioManager; this.pageSize = pageSize; this.maxFanIn = maxFanIn; this.channelManager = channelManager; this.compressionEnable = compressionEnable; this.compressionCodecFactory = compressionCodecFactory; this.compressionBlockSize = compressionBlockSize; }
Example #9
Source File: ReusingMergeOuterJoinIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public ReusingMergeOuterJoinIterator( OuterJoinType outerJoinType, MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); this.copy1 = serializer1.createInstance(); this.spillHeadCopy = serializer1.createInstance(); this.copy2 = serializer2.createInstance(); this.blockHeadCopy = serializer2.createInstance(); }
Example #10
Source File: ReusingMergeInnerJoinIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public ReusingMergeInnerJoinIterator( MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); this.copy1 = serializer1.createInstance(); this.spillHeadCopy = serializer1.createInstance(); this.copy2 = serializer2.createInstance(); this.blockHeadCopy = serializer2.createInstance(); }
Example #11
Source File: FileChannelUtil.java From flink with Apache License 2.0 | 6 votes |
public static AbstractChannelWriterOutputView createOutputView( IOManager ioManager, FileIOChannel.ID channel, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel); return new CompressedHeaderlessChannelWriterOutputView( bufferWriter, compressionCodecFactory, compressionBlockSize); } else { BlockChannelWriter<MemorySegment> blockWriter = ioManager.createBlockChannelWriter(channel); return new HeaderlessChannelWriterOutputView( blockWriter, Arrays.asList( allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize) ), segmentSize ); } }
Example #12
Source File: CompressedBlockChannelReader.java From flink with Apache License 2.0 | 6 votes |
public CompressedBlockChannelReader( IOManager ioManager, ID channel, LinkedBlockingQueue<MemorySegment> blockQueue, BlockCompressionFactory codecFactory, int preferBlockSize, int segmentSize) throws IOException { this.reader = ioManager.createBufferFileReader(channel, this); this.blockQueue = blockQueue; copyCompress = preferBlockSize > segmentSize * 2; int blockSize = copyCompress ? preferBlockSize : segmentSize; this.decompressor = codecFactory.getDecompressor(); cause = new AtomicReference<>(); if (copyCompress) { this.buf = new byte[blockSize]; this.bufWrapper = ByteBuffer.wrap(buf); } BlockCompressor compressor = codecFactory.getCompressor(); for (int i = 0; i < 2; i++) { MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(blockSize)]); reader.readInto(new NetworkBuffer(segment, this)); } }
Example #13
Source File: UnilateralSortMerger.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory, IOManager ioManager, MutableObjectIterator<E> input, AbstractInvokable parentTask, TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator, int numSortBuffers, int maxNumFileHandles, float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords, boolean objectReuseEnabled) throws IOException { this ( memoryManager, memory, ioManager, input, parentTask, serializerFactory, comparator, numSortBuffers, maxNumFileHandles, startSpillingFraction, noSpillingMemory, handleLargeRecords, objectReuseEnabled, new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING)); }
Example #14
Source File: UnilateralSortMerger.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Creates the spilling thread. * * @param exceptionHandler The exception handler to call for all exceptions. * @param queues The queues used to pass buffers between the threads. * @param parentTask The task that started this thread. If non-null, it is used to register this thread. * @param memManager The memory manager used to allocate buffers for the readers and writers. * @param ioManager The I/I manager used to instantiate readers and writers from. * @param serializer * @param comparator * @param sortReadMemory * @param writeMemory * @param maxNumFileHandles */ public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues, AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, TypeSerializer<E> serializer, TypeComparator<E> comparator, List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles) { super(exceptionHandler, "SortMerger spilling thread", queues, parentTask); this.memManager = memManager; this.ioManager = ioManager; this.serializer = serializer; this.comparator = comparator; this.mergeReadMemory = sortReadMemory; this.writeMemory = writeMemory; this.maxFanIn = maxNumFileHandles; this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1; }
Example #15
Source File: ReusingMergeOuterJoinIterator.java From flink with Apache License 2.0 | 6 votes |
public ReusingMergeOuterJoinIterator( OuterJoinType outerJoinType, MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); this.copy1 = serializer1.createInstance(); this.spillHeadCopy = serializer1.createInstance(); this.copy2 = serializer2.createInstance(); this.blockHeadCopy = serializer2.createInstance(); }
Example #16
Source File: TempBarrier.java From flink with Apache License 2.0 | 5 votes |
public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory, MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException { this.serializer = serializerFactory.getSerializer(); this.memManager = memManager; this.memory = new ArrayList<MemorySegment>(numPages); memManager.allocatePages(owner, this.memory, numPages); this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize()); this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer); }
Example #17
Source File: FileChannelStreamsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testCloseAndDeleteOutputView() { final IOManager ioManager = new IOManagerAsync(); try { MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true); List<MemorySegment> memory = new ArrayList<MemorySegment>(); memMan.allocatePages(new DummyInvokable(), memory, 4); FileIOChannel.ID channel = ioManager.createChannel(); BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel); FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize()); new StringValue("Some test text").write(out); // close for the first time, make sure all memory returns out.close(); assertTrue(memMan.verifyEmpty()); // close again, should not cause an exception out.close(); // delete, make sure file is removed out.closeAndDelete(); assertFalse(new File(channel.getPath()).exists()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } finally { ioManager.shutdown(); } }
Example #18
Source File: MutableHashTable.java From flink with Apache License 2.0 | 5 votes |
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBloomFilters) { this(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, DEFAULT_RECORD_LEN, useBloomFilters); }
Example #19
Source File: HashTableTest.java From flink with Apache License 2.0 | 5 votes |
/** * This tests the case where no additional partition buffers are used at the point when spilling * is triggered, testing that overflow bucket buffers are taken into account when deciding which * partition to spill. */ @Test public void testSpillingFreesOnlyOverflowSegments() { final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE; final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class); final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class); @SuppressWarnings("unchecked") final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class); try (final IOManager ioMan = new IOManagerAsync()) { final int pageSize = 32*1024; final int numSegments = 34; List<MemorySegment> memory = getMemory(numSegments, pageSize); MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>( serializer, serializer, buildComparator, probeComparator, pairComparator, memory, ioMan, 1, false); table.open(new ByteValueIterator(100000000), new ByteValueIterator(1)); table.close(); checkNoTempFilesRemain(ioMan); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #20
Source File: AbstractSortMergeOuterJoinIteratorITCase.java From flink with Apache License 2.0 | 5 votes |
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType, MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws Exception;
Example #21
Source File: KeyGroupStream.java From stateful-functions with Apache License 2.0 | 5 votes |
KeyGroupStream( TypeSerializer<T> serializer, IOManager ioManager, MemorySegmentPool memorySegmentPool) { this.serializer = Objects.requireNonNull(serializer); this.memoryPool = Objects.requireNonNull(memorySegmentPool); // SpillingBuffer requires at least 1 memory segment to be present at construction, otherwise it // fails // so we memorySegmentPool.ensureAtLeastOneSegmentPresent(); this.target = new SpillingBuffer(ioManager, memorySegmentPool, memorySegmentPool.getSegmentSize()); }
Example #22
Source File: NonReusingMergeInnerJoinIterator.java From flink with Apache License 2.0 | 5 votes |
public NonReusingMergeInnerJoinIterator( MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); }
Example #23
Source File: BinaryExternalSorter.java From flink with Apache License 2.0 | 5 votes |
public BinaryExternalSorter( final Object owner, MemoryManager memoryManager, long reservedMemorySize, IOManager ioManager, AbstractRowSerializer<BaseRow> inputSerializer, BinaryRowSerializer serializer, NormalizedKeyComputer normalizedKeyComputer, RecordComparator comparator, Configuration conf) throws IOException { this(owner, memoryManager, reservedMemorySize, ioManager, inputSerializer, serializer, normalizedKeyComputer, comparator, conf, AlgorithmOptions.SORT_SPILLING_THRESHOLD.defaultValue()); }
Example #24
Source File: NonReusingBuildSecondHashJoinIterator.java From flink with Apache License 2.0 | 5 votes |
public NonReusingBuildSecondHashJoinIterator( MutableObjectIterator<V1> firstInput, MutableObjectIterator<V2> secondInput, TypeSerializer<V1> serializer1, TypeComparator<V1> comparator1, TypeSerializer<V2> serializer2, TypeComparator<V2> comparator2, TypePairComparator<V1, V2> pairComparator, MemoryManager memManager, IOManager ioManager, AbstractInvokable ownerTask, double memoryFraction, boolean probeSideOuterJoin, boolean buildSideOuterJoin, boolean useBitmapFilters) throws MemoryAllocationException { this.memManager = memManager; this.firstInput = firstInput; this.secondInput = secondInput; this.probeSideSerializer = serializer1; if(useBitmapFilters && probeSideOuterJoin) { throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side"); } this.probeSideOuterJoin = probeSideOuterJoin; this.buildSideOuterJoin = buildSideOuterJoin; this.hashJoin = getHashJoin(serializer2, comparator2, serializer1, comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters); }
Example #25
Source File: AbstractSortMergeOuterJoinIteratorITCase.java From flink with Apache License 2.0 | 5 votes |
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType, MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws Exception;
Example #26
Source File: SpillingBuffer.java From flink with Apache License 2.0 | 5 votes |
public SpillingBuffer(IOManager ioManager, MemorySegmentSource memSource, int segmentSize) { super(memSource.nextSegment(), segmentSize, 0); this.fullSegments = new ArrayList<MemorySegment>(16); this.memorySource = memSource; this.ioManager = ioManager; }
Example #27
Source File: MutableHashTable.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer, TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator, TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager, boolean useBloomFilters) { this(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator, memorySegments, ioManager, DEFAULT_RECORD_LEN, useBloomFilters); }
Example #28
Source File: NonReusingMergeInnerJoinIterator.java From flink with Apache License 2.0 | 5 votes |
public NonReusingMergeInnerJoinIterator( MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); }
Example #29
Source File: BinaryHashPartition.java From flink with Apache License 2.0 | 5 votes |
/** * Spills this partition to disk and sets it up such that it continues spilling records that are * added to * it. The spilling process must free at least one buffer, either in the partition's record * buffers, or in * the memory segments for overflow buckets. * The partition immediately takes back one buffer to use it for further spilling. * * @param ioAccess The I/O manager to be used to create a writer to disk. * @param targetChannel The id of the target channel for this partition. * @return The number of buffers that were freed by spilling this partition. * @throws IOException Thrown, if the writing failed. */ int spillPartition( IOManager ioAccess, FileIOChannel.ID targetChannel, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { // sanity checks if (!isInMemory()) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition that has already been spilled."); } if (getNumOccupiedMemorySegments() < 2) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition with less than two buffers."); } // create the channel block writer and spill the current buffers // that keep the build side buffers current block, as it is most likely not full, yet // we return the number of blocks that become available this.buildSideChannel = FileChannelUtil.createBlockChannelWriter( ioAccess, targetChannel, bufferReturnQueue, compressionEnable, compressionCodecFactory, compressionBlockSize, memorySegmentSize); return this.buildSideWriteBuffer.spill(this.buildSideChannel); }
Example #30
Source File: AbstractMergeInnerJoinIterator.java From flink with Apache License 2.0 | 5 votes |
public AbstractMergeInnerJoinIterator( MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask); }