org.apache.flink.testutils.serialization.types.Util Java Examples

The following examples show how to use org.apache.flink.testutils.serialization.types.Util. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 
Example #2
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #3
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #4
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #5
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 
Example #6
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 
Example #7
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #8
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 
Example #9
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #10
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #11
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #12
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #13
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 
Example #14
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 
Example #15
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #16
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 
Example #17
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #18
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #19
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #20
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomTypes() {
	try {
		final int numTypes = 100000;

		// test with an odd buffer size to force many unaligned cases
		testSequenceOfTypes(Util.randomRecords(numTypes), 57);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #21
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #22
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 
Example #23
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #24
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #25
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example #26
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #27
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #28
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #29
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 
Example #30
Source File: PagedViewsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}