org.apache.flink.testutils.serialization.types.SerializationTestType Java Examples

The following examples show how to use org.apache.flink.testutils.serialization.types.SerializationTestType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 
Example #2
Source File: DeserializationUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 
Example #3
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #4
Source File: PagedViewsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 
Example #5
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 
Example #6
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 
Example #7
Source File: DeserializationUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 
Example #8
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #9
Source File: PagedViewsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 
Example #10
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 
Example #11
Source File: DeserializationUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 
Example #12
Source File: PagedViewsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 
Example #13
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #14
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 
Example #15
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 
Example #16
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
public void testUnconsumedBuffer(
		RecordSerializer<SerializationTestType> serializer,
		RecordDeserializer<SerializationTestType> deserializer,
		SerializationTestType record,
		int segmentSize,
		byte... leftOverBytes) throws Exception {
	try (ByteArrayOutputStream unconsumedBytes = new ByteArrayOutputStream()) {
		serializer.serializeRecord(record);

		BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer, segmentSize);

		if (serializer.copyToBufferBuilder(serializationResult.getBufferBuilder()).isFullBuffer()) {
			// buffer is full => start deserializing
			Buffer buffer = serializationResult.buildBuffer();
			writeBuffer(buffer.readOnlySlice().getNioBufferReadable(), unconsumedBytes);
			deserializer.setNextBuffer(buffer);
			assertUnconsumedBuffer(unconsumedBytes, deserializer.getUnconsumedBuffer());

			deserializer.getNextRecord(record.getClass().newInstance());

			// move buffers as long as necessary (for long records)
			while ((serializationResult = setNextBufferForSerializer(serializer, segmentSize)).isFullBuffer()) {
				buffer = serializationResult.buildBuffer();

				if (serializationResult.isFullRecord()) {
					buffer = appendLeftOverBytes(buffer, leftOverBytes);
				}

				writeBuffer(buffer.readOnlySlice().getNioBufferReadable(), unconsumedBytes);
				deserializer.setNextBuffer(buffer);
				assertUnconsumedBuffer(unconsumedBytes, deserializer.getUnconsumedBuffer());

				deserializer.getNextRecord(record.getClass().newInstance());
			}
		}
	}
}
 
Example #17
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 
Example #18
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 
Example #19
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 
Example #20
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 
Example #21
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 
Example #22
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #23
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #24
Source File: BroadcastRecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * FLINK-17780: Tests that a shared buffer(or memory segment) of a buffer builder is only freed when all consumers
 * are closed.
 */
@Test
public void testRandomEmitAndBufferRecycling() throws Exception {
	int recordSize = 8;

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(2, 2 * recordSize);
	final KeepingPartitionWriter partitionWriter = new KeepingPartitionWriter(bufferProvider) {
		@Override
		public int getNumberOfSubpartitions() {
			return 2;
		}
	};
	final BroadcastRecordWriter<SerializationTestType> writer = new BroadcastRecordWriter<>(partitionWriter, 0, "test");

	// force materialization of both buffers for easier availability tests
	List<Buffer> buffers = Arrays.asList(bufferProvider.requestBuffer(), bufferProvider.requestBuffer());
	buffers.forEach(Buffer::recycleBuffer);
	assertEquals(2, bufferProvider.getNumberOfAvailableBuffers());

	// fill first buffer
	writer.randomEmit(new IntType(1), 0);
	writer.broadcastEmit(new IntType(2));
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());

	// simulate consumption of first buffer consumer; this should not free buffers
	assertEquals(1, partitionWriter.getAddedBufferConsumers(0).size());
	closeConsumer(partitionWriter, 0, 2 * recordSize);
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());

	// use second buffer
	writer.broadcastEmit(new IntType(3));
	assertEquals(0, bufferProvider.getNumberOfAvailableBuffers());

	// fully free first buffer
	assertEquals(2, partitionWriter.getAddedBufferConsumers(1).size());
	closeConsumer(partitionWriter, 1, recordSize);
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());
}
 
Example #25
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #26
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 
Example #27
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 
Example #28
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 
Example #29
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void verifyDeserializationResults(
		Queue<BufferConsumer> queue,
		RecordDeserializer<SerializationTestType> deserializer,
		ArrayDeque<SerializationTestType> expectedRecords,
		int numRequiredBuffers,
		int numValues) throws Exception {
	int assertRecords = 0;
	for (int j = 0; j < numRequiredBuffers; j++) {
		Buffer buffer = buildSingleBuffer(queue.remove());
		deserializer.setNextBuffer(buffer);

		assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
	}
	Assert.assertEquals(numValues, assertRecords);
}
 
Example #30
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testEmptyRecords() throws IOException {
	final int segmentSize = 11;

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final BufferBuilder bufferBuilder1 = createBufferBuilder(segmentSize);

	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD,
		serializer.copyToBufferBuilder(bufferBuilder1));

	SerializationTestType emptyRecord = new SerializationTestType() {
		@Override
		public SerializationTestType getRandom(Random rnd) {
			throw new UnsupportedOperationException();
		}

		@Override
		public int length() {
			throw new UnsupportedOperationException();
		}

		@Override
		public void write(DataOutputView out) {}

		@Override
		public void read(DataInputView in) {}

		@Override
		public int hashCode() {
			throw new UnsupportedOperationException();
		}

		@Override
		public boolean equals(Object obj) {
			throw new UnsupportedOperationException();
		}
	};

	serializer.serializeRecord(emptyRecord);
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.copyToBufferBuilder(bufferBuilder1));

	serializer.reset();
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.copyToBufferBuilder(bufferBuilder1));

	serializer.reset();
	Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL,
		serializer.copyToBufferBuilder(bufferBuilder1));

	final BufferBuilder bufferBuilder2 = createBufferBuilder(segmentSize);
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD,
		serializer.copyToBufferBuilder(bufferBuilder2));
}