Java Code Examples for org.apache.flink.testutils.serialization.types.Util

The following examples show how to use org.apache.flink.testutils.serialization.types.Util. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example 2
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example 3
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 
Example 4
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example 5
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 6
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 7
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 8
@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 10
Source Project: Flink-CEPplus   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomTypes() {
	try {
		final int numTypes = 100000;

		// test with an odd buffer size to force many unaligned cases
		testSequenceOfTypes(Util.randomRecords(numTypes), 57);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 12
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 13
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 14
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 15
@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 
Example 16
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example 17
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 18
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 19
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 20
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 
Example 21
Source Project: flink   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 22
Source Project: flink   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 23
Source Project: flink   Source File: PagedViewsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomTypes() {
	try {
		final int numTypes = 100000;

		// test with an odd buffer size to force many unaligned cases
		testSequenceOfTypes(Util.randomRecords(numTypes), 57);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example 24
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 25
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 26
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 27
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 
Example 28
Source Project: flink   Source File: SpanningRecordSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 
Example 29
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example 30
Source Project: flink   Source File: SpanningRecordSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}