Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializer#serialize()

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializer#serialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SerializedCheckpointData.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Converts a list of checkpoints into an array of SerializedCheckpointData.
 *
 * @param checkpoints The checkpoints to be converted into IdsCheckpointData.
 * @param serializer The serializer to serialize the IDs.
 * @param outputBuffer The reusable serialization buffer.
 * @param <T> The type of the ID.
 * @return An array of serializable SerializedCheckpointData, one per entry in the queue.
 *
 * @throws IOException Thrown, if the serialization fails.
 */
public static <T> SerializedCheckpointData[] fromDeque(
		ArrayDeque<Tuple2<Long, Set<T>>> checkpoints,
		TypeSerializer<T> serializer,
		DataOutputSerializer outputBuffer) throws IOException {

	SerializedCheckpointData[] serializedCheckpoints = new SerializedCheckpointData[checkpoints.size()];

	int pos = 0;
	for (Tuple2<Long, Set<T>> checkpoint : checkpoints) {
		outputBuffer.clear();
		Set<T> checkpointIds = checkpoint.f1;

		for (T id : checkpointIds) {
			serializer.serialize(id, outputBuffer);
		}

		serializedCheckpoints[pos++] = new SerializedCheckpointData(
				checkpoint.f0, outputBuffer.getCopyOfBuffer(), checkpointIds.size());
	}

	return serializedCheckpoints;
}
 
Example 2
Source File: RocksDBListState.java    From flink with Apache License 2.0 6 votes vote down vote up
byte[] serializeValueList(
	List<T> valueList,
	TypeSerializer<T> elementSerializer,
	@SuppressWarnings("SameParameterValue") byte delimiter) throws IOException {

	out.clear();
	boolean first = true;

	for (T value : valueList) {
		Preconditions.checkNotNull(value, "You cannot add null to a value list.");

		if (first) {
			first = false;
		} else {
			out.write(delimiter);
		}
		elementSerializer.serialize(value, out);
	}

	return out.getCopyOfBuffer();
}
 
Example 3
Source File: ImmutableListStateTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example 4
Source File: AbstractRocksDBState.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
<T> byte[] serializeValueList(
	List<T> valueList,
	TypeSerializer<T> elementSerializer,
	byte delimiter) throws IOException {

	dataOutputView.clear();
	boolean first = true;

	for (T value : valueList) {
		Preconditions.checkNotNull(value, "You cannot add null to a value list.");

		if (first) {
			first = false;
		} else {
			dataOutputView.write(delimiter);
		}
		elementSerializer.serialize(value, dataOutputView);
	}

	return dataOutputView.getCopyOfBuffer();
}
 
Example 5
Source File: KvStateSerializer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Serializes the value with the given serializer.
 *
 * @param value      Value of type T to serialize
 * @param serializer Serializer for T
 * @param <T>        Type of the value
 * @return Serialized value or <code>null</code> if value <code>null</code>
 * @throws IOException On failure during serialization
 */
public static <T> byte[] serializeValue(T value, TypeSerializer<T> serializer) throws IOException {
	if (value != null) {
		// Serialize
		DataOutputSerializer dos = new DataOutputSerializer(32);
		serializer.serialize(value, dos);
		return dos.getCopyOfBuffer();
	} else {
		return null;
	}
}
 
Example 6
Source File: FixedLengthRecordSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 
Example 7
Source File: KvStateSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Serializes the value with the given serializer.
 *
 * @param value      Value of type T to serialize
 * @param serializer Serializer for T
 * @param <T>        Type of the value
 * @return Serialized value or <code>null</code> if value <code>null</code>
 * @throws IOException On failure during serialization
 */
public static <T> byte[] serializeValue(T value, TypeSerializer<T> serializer) throws IOException {
	if (value != null) {
		// Serialize
		DataOutputSerializer dos = new DataOutputSerializer(32);
		serializer.serialize(value, dos);
		return dos.getCopyOfBuffer();
	} else {
		return null;
	}
}
 
Example 8
Source File: RocksDBKeySerializationUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <K> void writeKey(
	K key,
	TypeSerializer<K> keySerializer,
	DataOutputSerializer keySerializationDataOutputView,
	boolean ambiguousKeyPossible) throws IOException {
	//write key
	int beforeWrite = keySerializationDataOutputView.length();
	keySerializer.serialize(key, keySerializationDataOutputView);

	if (ambiguousKeyPossible) {
		//write size of key
		writeLengthFrom(beforeWrite, keySerializationDataOutputView);
	}
}
 
Example 9
Source File: AvroRecordInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 
Example 10
Source File: FixedLengthRecordSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Writes the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int recordsLeft = this.numRecords;
	int currentMemSeg = 0;
	
	while (recordsLeft > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, 0);
		
		// check whether we have a full or partially full segment
		if (recordsLeft >= recordsPerSegment) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			recordsLeft -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; recordsLeft > 0; recordsLeft--) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}
	}
}
 
Example 11
Source File: CollectTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <T> List<byte[]> toBytesList(List<T> values, TypeSerializer<T> serializer) {
	List<byte[]> ret = new ArrayList<>();
	for (T value : values) {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);
		try {
			serializer.serialize(value, wrapper);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
		ret.add(baos.toByteArray());
	}
	return ret;
}
 
Example 12
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 
Example 13
Source File: FixedLengthRecordSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 
Example 14
Source File: PojoSerializationBenchmark.java    From flink-benchmarks with Apache License 2.0 4 votes vote down vote up
private <T> byte[] write(TypeSerializer<T> serializer, T value) throws IOException {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputView out = new DataOutputViewStreamWrapper(buffer);
    serializer.serialize(value, out);
    return buffer.toByteArray();
}
 
Example 15
Source File: ChannelViewsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testReadTooMany() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());

	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();

	// read and re-generate all records and compare them
	try {
		final Tuple2<Integer, String> readRec = new Tuple2<>();
		for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
			generator.next(rec);
			serializer.deserialize(readRec, inView);
			final int k1 = rec.f0;
			final String v1 = rec.f1;
			final int k2 = readRec.f0;
			final String v2 = readRec.f1;
			Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
		}
		Assert.fail("Expected an EOFException which did not occur.");
	}
	catch (EOFException eofex) {
		// expected
	}
	catch (Throwable t) {
		// unexpected
		Assert.fail("Unexpected Exception: " + t.getMessage());
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 
Example 16
Source File: SocketStreamIteratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testIterator() throws Exception {

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final long seed = new Random().nextLong();
	final int numElements = 1000;

	final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);

	Thread writer = new Thread() {

		@Override
		public void run() {
			try {
				try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
					DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {

					final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
					final Random rnd = new Random(seed);

					for (int i = 0; i < numElements; i++) {
						serializer.serialize(rnd.nextLong(), out);
					}
				}
			}
			catch (Throwable t) {
				error.set(t);
			}
		}
	};

	writer.start();

	final Random validator = new Random(seed);
	for (int i = 0; i < numElements; i++) {
		assertTrue(iterator.hasNext());
		assertTrue(iterator.hasNext());
		assertEquals(validator.nextLong(), iterator.next().longValue());
	}

	assertFalse(iterator.hasNext());
	writer.join();
	assertFalse(iterator.hasNext());
}
 
Example 17
Source File: AvroSerializerSnapshotTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private static <T> ByteBuffer serialize(TypeSerializer<T> serializer, T record) throws IOException {
	DataOutputSerializer out = new DataOutputSerializer(1024);
	serializer.serialize(record, out);
	return out.wrapAsByteBuffer();
}
 
Example 18
Source File: SocketStreamIteratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testIterator() throws Exception {

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final long seed = new Random().nextLong();
	final int numElements = 1000;

	final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);

	Thread writer = new Thread() {

		@Override
		public void run() {
			try {
				try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
					DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {

					final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
					final Random rnd = new Random(seed);

					for (int i = 0; i < numElements; i++) {
						serializer.serialize(rnd.nextLong(), out);
					}
				}
			}
			catch (Throwable t) {
				error.set(t);
			}
		}
	};

	writer.start();

	final Random validator = new Random(seed);
	for (int i = 0; i < numElements; i++) {
		assertTrue(iterator.hasNext());
		assertTrue(iterator.hasNext());
		assertEquals(validator.nextLong(), iterator.next().longValue());
	}

	assertFalse(iterator.hasNext());
	writer.join();
	assertFalse(iterator.hasNext());
}
 
Example 19
Source File: KvStateSerializer.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Serializes the key and namespace into a {@link ByteBuffer}.
 *
 * <p>The serialized format matches the RocksDB state backend key format, i.e.
 * the key and namespace don't have to be deserialized for RocksDB lookups.
 *
 * @param key                 Key to serialize
 * @param keySerializer       Serializer for the key
 * @param namespace           Namespace to serialize
 * @param namespaceSerializer Serializer for the namespace
 * @param <K>                 Key type
 * @param <N>                 Namespace type
 * @return Buffer holding the serialized key and namespace
 * @throws IOException Serialization errors are forwarded
 */
public static <K, N> byte[] serializeKeyAndNamespace(
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer) throws IOException {

	DataOutputSerializer dos = new DataOutputSerializer(32);

	keySerializer.serialize(key, dos);
	dos.writeByte(MAGIC_NUMBER);
	namespaceSerializer.serialize(namespace, dos);

	return dos.getCopyOfBuffer();
}
 
Example 20
Source File: SerializeUtil.java    From flink-spector with Apache License 2.0 3 votes vote down vote up
/**
 * Serialize an object using a TypeSerializer.
 *
 * @param object     to serialize
 * @param serializer to use;
 * @param <IN>       type of the object
 * @return serialized object
 * @throws IOException
 */
public static <IN> byte[] serialize(IN object, TypeSerializer<IN> serializer) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(new DataOutputStream(baos));
    serializer.serialize(object, wrapper);
    return baos.toByteArray();
}