org.apache.flink.testutils.serialization.types.SerializationTestTypeFactory Java Examples

The following examples show how to use org.apache.flink.testutils.serialization.types.SerializationTestTypeFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 
Example #2
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #3
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #4
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 
Example #5
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #6
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #7
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #8
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #9
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #10
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #11
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 
Example #12
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 
Example #13
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 
Example #14
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #15
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #16
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #17
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #18
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 
Example #19
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #20
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #21
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #22
Source File: PagedViewsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 
Example #23
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #24
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #25
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #26
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #27
Source File: SpanningRecordSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #28
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #29
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 
Example #30
Source File: SpanningRecordSerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}