Java Code Examples for org.apache.flink.runtime.memory.MemoryManager#getPageSize()

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

@Override
public void open() throws Exception {
	super.open();

	ClassLoader cl = getUserCodeClassloader();
	serializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(cl);
	partitionComparator = genComparator.newInstance(cl);
	genComparator = null;

	MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
	LazyMemorySegmentPool pool = new LazyMemorySegmentPool(
			this,
			memManager,
			(int) (computeMemorySize() / memManager.getPageSize()));
	this.currentData = new ResettableExternalBuffer(
			getContainingTask().getEnvironment().getIOManager(),
			pool,
			serializer, isRowAllInFixedPart);

	collector = new StreamRecordCollector<>(output);
	joinedRows = new JoinedRowData[overWindowFrames.length];
	for (int i = 0; i < overWindowFrames.length; i++) {
		overWindowFrames[i].open(new ExecutionContextImpl(this, getRuntimeContext()));
		joinedRows[i] = new JoinedRowData();
	}
}
 

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 

@Test
public void testCloseAndDeleteOutputView() {
	try (IOManager ioManager = new IOManagerAsync()) {
		MemoryManager memMan = MemoryManagerBuilder.newBuilder().build();
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public FileChannelInputView(BlockChannelReader<MemorySegment> reader, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(reader);
	checkNotNull(memManager);
	checkNotNull(memory);
	checkArgument(!reader.isClosed());
	checkArgument(memory.size() > 0);
	
	this.reader = reader;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	
	try {
		final long channelLength = reader.getSize();
		final int segmentSize = memManager.getPageSize();
		
		this.numBlocksRemaining = MathUtils.checkedDownCast(channelLength / segmentSize);
		if (channelLength % segmentSize != 0) {
			this.numBlocksRemaining++;
		}
		
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

@Test
public void testCloseAndDeleteOutputView() {
	try (IOManager ioManager = new IOManagerAsync()) {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public BaseHybridHashTable(
		Configuration conf,
		Object owner,
		MemoryManager memManager,
		long reservedMemorySize,
		long preferredMemorySize,
		long perRequestMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount,
		boolean tryDistinctBuildRow) {

	//TODO: read compression config from configuration
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
			? BlockCompressionFactory.createBlockCompressionFactory(BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
			: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();

	this.owner = owner;
	this.avgRecordLen = avgRecordLen;
	this.buildRowCount = buildRowCount;
	this.tryDistinctBuildRow = tryDistinctBuildRow;
	this.reservedNumBuffers = (int) (reservedMemorySize / memManager.getPageSize());
	// some sanity checks first
	checkArgument(reservedNumBuffers >= MIN_NUM_MEMORY_SEGMENTS);
	this.maxNumBuffers = (int) (preferredMemorySize / memManager.getPageSize());
	this.perRequestNumBuffers = (int) (perRequestMemorySize / memManager.getPageSize());
	this.availableMemory = new ArrayList<>(this.reservedNumBuffers);
	try {
		List<MemorySegment> allocates = memManager.allocatePages(owner, this.reservedNumBuffers);
		this.availableMemory.addAll(allocates);
		allocates.clear();
	} catch (MemoryAllocationException e) {
		LOG.error("Out of memory", e);
		throw new RuntimeException(e);
	}
	this.memManager = memManager;
	this.ioManager = ioManager;

	this.segmentSize = memManager.getPageSize();

	checkArgument(MathUtils.isPowerOf2(segmentSize));

	// take away the write behind buffers
	this.buildSpillReturnBuffers = new LinkedBlockingQueue<>();

	this.segmentSizeBits = MathUtils.log2strict(segmentSize);
	this.segmentSizeMask = segmentSize - 1;

	// open builds the initial table by consuming the build-side input
	this.currentRecursionDepth = 0;

	// create the partitions
	this.initPartitionFanOut = Math.min(getPartitioningFanOutNoEstimates(), maxNumPartition());

	this.closed.set(false);

	LOG.info(String.format("Initialize hash table with %d memory segments, each size [%d], the reserved memory %d" +
					" MB, the max memory %d MB, per allocate {} segments from floating memory pool.",
			reservedNumBuffers, segmentSize, (long) reservedNumBuffers * segmentSize / 1024 / 1024,
			(long) maxNumBuffers * segmentSize / 1024 / 1024), perRequestNumBuffers);
}
 

public BytesHashMap(
		final Object owner,
		MemoryManager memoryManager,
		long memorySize,
		LogicalType[] keyTypes,
		LogicalType[] valueTypes,
		boolean inferBucketMemory) {
	this.segmentSize = memoryManager.getPageSize();
	this.reservedNumBuffers = (int) (memorySize / segmentSize);
	this.memoryPool = new LazyMemorySegmentPool(owner, memoryManager, reservedNumBuffers);
	this.numBucketsPerSegment = segmentSize / BUCKET_SIZE;
	this.numBucketsPerSegmentBits = MathUtils.log2strict(this.numBucketsPerSegment);
	this.numBucketsPerSegmentMask = (1 << this.numBucketsPerSegmentBits) - 1;
	this.lastBucketPosition = (numBucketsPerSegment - 1) * BUCKET_SIZE;

	checkArgument(keyTypes.length > 0);
	this.keySerializer = new BinaryRowDataSerializer(keyTypes.length);
	this.reusedKey = this.keySerializer.createInstance();

	if (valueTypes.length == 0) {
		this.valueSerializer = new BinaryRowDataSerializer(0);
		this.hashSetMode = true;
		this.reusedValue = new BinaryRowData(0);
		this.reusedValue.pointTo(MemorySegmentFactory.wrap(new byte[8]), 0, 8);
		LOG.info("BytesHashMap with hashSetMode = true.");
	} else {
		this.valueSerializer = new BinaryRowDataSerializer(valueTypes.length);
		this.hashSetMode = false;
		this.reusedValue = this.valueSerializer.createInstance();
	}

	this.reuseLookInfo = new LookupInfo();

	this.recordArea = new RecordArea();

	int initBucketSegmentNum;
	if (inferBucketMemory) {
		initBucketSegmentNum = calcNumBucketSegments(keyTypes, valueTypes);
	} else {
		checkArgument(memorySize > INIT_BUCKET_MEMORY_IN_BYTES, "The minBucketMemorySize is not valid!");
		initBucketSegmentNum = MathUtils.roundDownToPowerOf2((int) (INIT_BUCKET_MEMORY_IN_BYTES / segmentSize));
	}

	// allocate and initialize MemorySegments for bucket area
	initBucketSegments(initBucketSegmentNum);

	LOG.info("BytesHashMap with initial memory segments {}, {} in bytes, init allocating {} for bucket area.",
			reservedNumBuffers, reservedNumBuffers * segmentSize, initBucketSegmentNum);
}
 

public BaseHybridHashTable(
		Configuration conf,
		Object owner,
		MemoryManager memManager,
		long reservedMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount,
		boolean tryDistinctBuildRow) {

	//TODO: read compression config from configuration
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
			? BlockCompressionFactory.createBlockCompressionFactory(BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
			: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();

	this.avgRecordLen = avgRecordLen;
	this.buildRowCount = buildRowCount;
	this.tryDistinctBuildRow = tryDistinctBuildRow;
	this.totalNumBuffers = (int) (reservedMemorySize / memManager.getPageSize());
	// some sanity checks first
	checkArgument(totalNumBuffers >= MIN_NUM_MEMORY_SEGMENTS);
	this.internalPool = new LazyMemorySegmentPool(owner, memManager, totalNumBuffers);
	this.ioManager = ioManager;

	this.segmentSize = memManager.getPageSize();

	checkArgument(MathUtils.isPowerOf2(segmentSize));

	// take away the write behind buffers
	this.buildSpillReturnBuffers = new LinkedBlockingQueue<>();

	this.segmentSizeBits = MathUtils.log2strict(segmentSize);
	this.segmentSizeMask = segmentSize - 1;

	// open builds the initial table by consuming the build-side input
	this.currentRecursionDepth = 0;

	// create the partitions
	this.initPartitionFanOut = Math.min(getPartitioningFanOutNoEstimates(), maxNumPartition());

	this.closed.set(false);

	LOG.info(String.format("Initialize hash table with %d memory segments, each size [%d], the memory %d MB.",
			totalNumBuffers, segmentSize, (long) totalNumBuffers * segmentSize / 1024 / 1024));
}