Java Code Examples for org.apache.flink.core.memory.MemorySegmentFactory#allocateUnpooledSegment()

The following examples show how to use org.apache.flink.core.memory.MemorySegmentFactory#allocateUnpooledSegment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AsynchronousBufferFileWriterTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddWithFailingWriter() throws Exception {
	AsynchronousBufferFileWriter writer =
		new AsynchronousBufferFileWriter(ioManager.createChannel(), new RequestQueue<>());
	writer.close();

	exception.expect(IOException.class);

	Buffer buffer = new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(4096),
		FreeingBufferRecycler.INSTANCE);
	try {
		writer.writeBlock(buffer);
	} finally {
		if (!buffer.isRecycled()) {
			buffer.recycleBuffer();
			Assert.fail("buffer not recycled");
		}
		assertEquals("Shouln't increment number of outstanding requests.", 0, writer.getNumberOfOutstandingRequests());
	}
}
Example 2
Source File: MemorySegmentSimpleTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testByteBufferWrapping() {
	try {
		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(1024);

		ByteBuffer buf1 = seg.wrap(13, 47);
		assertEquals(13, buf1.position());
		assertEquals(60, buf1.limit());
		assertEquals(47, buf1.remaining());

		ByteBuffer buf2 = seg.wrap(500, 267);
		assertEquals(500, buf2.position());
		assertEquals(767, buf2.limit());
		assertEquals(267, buf2.remaining());

		ByteBuffer buf3 = seg.wrap(0, 1024);
		assertEquals(0, buf3.position());
		assertEquals(1024, buf3.limit());
		assertEquals(1024, buf3.remaining());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example 3
Source File: CheckpointBarrierAlignerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel) {
	final int size = sizeCounter++;
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 4
Source File: BinaryStringTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void compareTo() {
	assertEquals(0, fromString("   ").compareTo(blankString(3)));
	assertTrue(fromString("").compareTo(fromString("a")) < 0);
	assertTrue(fromString("abc").compareTo(fromString("ABC")) > 0);
	assertTrue(fromString("abc0").compareTo(fromString("abc")) > 0);
	assertEquals(0, fromString("abcabcabc").compareTo(fromString("abcabcabc")));
	assertTrue(fromString("aBcabcabc").compareTo(fromString("Abcabcabc")) > 0);
	assertTrue(fromString("Abcabcabc").compareTo(fromString("abcabcabC")) < 0);
	assertTrue(fromString("abcabcabc").compareTo(fromString("abcabcabC")) > 0);

	assertTrue(fromString("abc").compareTo(fromString("世界")) < 0);
	assertTrue(fromString("你好").compareTo(fromString("世界")) > 0);
	assertTrue(fromString("你好123").compareTo(fromString("你好122")) > 0);

	MemorySegment segment1 = MemorySegmentFactory.allocateUnpooledSegment(1024);
	MemorySegment segment2 = MemorySegmentFactory.allocateUnpooledSegment(1024);
	SortUtil.putStringNormalizedKey(fromString("abcabcabc"), segment1, 0, 9);
	SortUtil.putStringNormalizedKey(fromString("abcabcabC"), segment2, 0, 9);
	assertTrue(segment1.compare(segment2, 0, 0, 9) > 0);
	SortUtil.putStringNormalizedKey(fromString("abcab"), segment1, 0, 9);
	assertTrue(segment1.compare(segment2, 0, 0, 9) < 0);
}
Example 5
Source File: AsynchronousBufferFileWriterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddWithFailingWriter() throws Exception {
	AsynchronousBufferFileWriter writer =
		new AsynchronousBufferFileWriter(ioManager.createChannel(), new RequestQueue<>());
	writer.close();

	exception.expect(IOException.class);

	Buffer buffer = new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(4096),
		FreeingBufferRecycler.INSTANCE);
	try {
		writer.writeBlock(buffer);
	} finally {
		if (!buffer.isRecycled()) {
			buffer.recycleBuffer();
			Assert.fail("buffer not recycled");
		}
		assertEquals("Shouln't increment number of outstanding requests.", 0, writer.getNumberOfOutstandingRequests());
	}
}
Example 6
Source File: MemorySegmentSimpleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testByteBufferWrapping() {
	try {
		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(1024);

		ByteBuffer buf1 = seg.wrap(13, 47);
		assertEquals(13, buf1.position());
		assertEquals(60, buf1.limit());
		assertEquals(47, buf1.remaining());

		ByteBuffer buf2 = seg.wrap(500, 267);
		assertEquals(500, buf2.position());
		assertEquals(767, buf2.limit());
		assertEquals(267, buf2.remaining());

		ByteBuffer buf3 = seg.wrap(0, 1024);
		assertEquals(0, buf3.position());
		assertEquals(1024, buf3.limit());
		assertEquals(1024, buf3.remaining());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example 7
Source File: BarrierBufferTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel, int pageSize) {
	final int size = sizeCounter++;
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(pageSize);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 8
Source File: ReadOnlySlicedBufferTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE);
	buffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, Buffer.DataType.DATA_BUFFER, 0);
	for (int i = 0; i < DATA_SIZE; ++i) {
		buffer.writeByte(i);
	}
}
Example 9
Source File: BufferBlockerTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static BufferOrEvent generateRandomBuffer(int size, int channelIndex) {
	MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	for (int i = 0; i < size; i++) {
		seg.put(i, (byte) i);
	}

	Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);
	return new BufferOrEvent(buf, channelIndex);
}
Example 10
Source File: TestBufferFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
public synchronized Buffer create() {
	if (numberOfCreatedBuffers >= poolSize) {
		return null;
	}

	numberOfCreatedBuffers++;
	return new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), bufferRecycler);
}
Example 11
Source File: BufferBuilderTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static BufferBuilder createFilledBufferBuilder(int size, int dataSize) {
	checkArgument(size >= dataSize);
	BufferBuilder bufferBuilder = new BufferBuilder(
		MemorySegmentFactory.allocateUnpooledSegment(size),
		FreeingBufferRecycler.INSTANCE);
	return fillBufferBuilder(bufferBuilder, dataSize);
}
Example 12
Source File: ReadOnlySlicedBufferTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE);
	buffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, true, 0);
	for (int i = 0; i < DATA_SIZE; ++i) {
		buffer.writeByte(i);
	}
}
Example 13
Source File: BufferSpiller.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel, true);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel, true, length);
	}
}
Example 14
Source File: BufferBuilderTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static BufferConsumer createEventBufferConsumer(int size) {
	return new BufferConsumer(
		MemorySegmentFactory.allocateUnpooledSegment(size),
		FreeingBufferRecycler.INSTANCE,
		false);
}
Example 15
Source File: BufferBuilderTestUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static BufferConsumer createEventBufferConsumer(int size) {
	return new BufferConsumer(
		MemorySegmentFactory.allocateUnpooledSegment(size),
		FreeingBufferRecycler.INSTANCE,
		false);
}
Example 16
Source File: BufferFileWriterReaderTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Buffer createBuffer() {
	return new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE), BUFFER_RECYCLER);
}
Example 17
Source File: PagedViewsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
protected MemorySegment nextSegment(MemorySegment current, int positionInCurrent) throws IOException {
	segments.add(new SegmentWithPosition(current, positionInCurrent));
	return MemorySegmentFactory.allocateUnpooledSegment(segmentSize);
}
Example 18
Source File: BufferBuilderTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static BufferConsumer createEventBufferConsumer(int size, Buffer.DataType dataType) {
	return new BufferConsumer(
		MemorySegmentFactory.allocateUnpooledSegment(size),
		FreeingBufferRecycler.INSTANCE,
		dataType);
}
Example 19
Source File: BufferFileWriterFileSegmentReaderTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private Buffer createBuffer() {
	return new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE), BUFFER_RECYCLER);
}
Example 20
Source File: BufferSpiller.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel);
	}
}