Java Code Examples for org.apache.flink.core.memory.DataOutputViewStreamWrapper

The following examples show how to use org.apache.flink.core.memory.DataOutputViewStreamWrapper. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: TypeSerializerSerializationUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that reading and writing serializers work correctly.
 */
@Test
public void testSerializerSerialization() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(serializer, deserializedSerializer);
}
 
Example 2
Source Project: flink   Source File: VersionedIOWriteableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
 
Example 3
Source Project: flink   Source File: FailingCollectionSource.java    License: Apache License 2.0 6 votes vote down vote up
public FailingCollectionSource(
	TypeSerializer<T> serializer,
	Iterable<T> elements,
	int failureAfterNumElements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
	checkArgument(failureAfterNumElements > 0);
	this.failureAfterNumElements = failureAfterNumElements;
	this.checkpointedEmittedNums = new HashMap<>();
}
 
Example 4
Source Project: Flink-CEPplus   Source File: InstantiationUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
	if (original == null) {
		return null;
	}

	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		original.write(out);
	}

	final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {

		@SuppressWarnings("unchecked")
		T copy = (T) instantiate(original.getClass());
		copy.read(in);
		return copy;
	}
}
 
Example 5
Source Project: flink   Source File: VersionedIOWriteableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
 
Example 6
Source Project: flink   Source File: SerializationProxiesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example 7
Source Project: flink   Source File: FromElementsFunction.java    License: Apache License 2.0 6 votes vote down vote up
public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
}
 
Example 8
Source Project: flink   Source File: ImmutableListStateTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example 9
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example 10
Source Project: flink   Source File: FoldApplyProcessAllWindowFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example 11
Source Project: flink   Source File: RocksDBKeySerializationUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example 12
Source Project: flink   Source File: InternalPriorityQueueTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int compare(TestElement o1, TestElement o2) {

	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os);
	try {
		TestElementSerializer.INSTANCE.serialize(o1, ow);
		byte[] a1 = os.toByteArray();
		os.reset();
		TestElementSerializer.INSTANCE.serialize(o2, ow);
		byte[] a2 = os.toByteArray();
		return UnsignedBytes.lexicographicalComparator().compare(a1, a2);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
Example 13
Source Project: flink   Source File: FoldApplyWindowFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	// out type is not used, just use this for the execution config
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example 14
Source Project: Flink-CEPplus   Source File: VersionedIOWriteableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
 
Example 15
Source Project: Flink-CEPplus   Source File: VersionedIOWriteableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
 
Example 16
Source Project: flink   Source File: TypeSerializerSerializationUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example 17
Source Project: flink   Source File: ImmutableListStateTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example 18
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example 19
Source Project: flink   Source File: CopyOnWriteStateTableTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This tests that resource can be released for a successful snapshot.
 */
@Test
public void testReleaseForSuccessfulSnapshot() throws IOException {
	int numberOfKeyGroups = 10;
	CopyOnWriteStateTable<Integer, Integer, Float> table = createStateTableForSnapshotRelease(numberOfKeyGroups);

	ByteArrayOutputStreamWithPos byteArrayOutputStreamWithPos = new ByteArrayOutputStreamWithPos();
	DataOutputView dataOutputView = new DataOutputViewStreamWrapper(byteArrayOutputStreamWithPos);

	CopyOnWriteStateTableSnapshot<Integer, Integer, Float> snapshot = table.stateSnapshot();
	for (int group = 0; group < numberOfKeyGroups; group++) {
		snapshot.writeStateInKeyGroup(dataOutputView, group);
		// resource used by one key group should be released after the snapshot is successful
		Assert.assertTrue(isResourceReleasedForKeyGroup(table, group));
	}
	snapshot.release();
	verifyResourceIsReleasedForAllKeyGroup(table, 1);
}
 
Example 20
Source Project: flink   Source File: TypeSerializerSerializationUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example 21
Source Project: Flink-CEPplus   Source File: EventWithAggregatorsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example 22
Source Project: flink   Source File: EventWithAggregatorsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example 23
@Test
public void testReadWriteMissingKeyGroups() throws Exception {
	final KeyGroupRange keyRange = new KeyGroupRange(0, 2);
	KeyedStateCheckpointOutputStream stream = createStream(keyRange);

	DataOutputView dov = new DataOutputViewStreamWrapper(stream);
	stream.startNewKeyGroup(1);
	dov.writeInt(1);

	KeyGroupsStateHandle fullHandle = stream.closeAndGetHandle();

	int count = 0;
	try (FSDataInputStream in = fullHandle.openInputStream()) {
		DataInputView div = new DataInputViewStreamWrapper(in);
		for (int kg : fullHandle.getKeyGroupRange()) {
			long off = fullHandle.getOffsetForKeyGroup(kg);
			if (off >= 0) {
				in.seek(off);
				Assert.assertEquals(1, div.readInt());
				++count;
			}
		}
	}

	Assert.assertEquals(1, count);
}
 
Example 24
Source Project: flink   Source File: RocksFullSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 
Example 25
Source Project: Flink-CEPplus   Source File: StreamGroupedFold.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	outTypeSerializer = outTypeInfo.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		outTypeSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
				initialValue.getClass().getSimpleName() + " of fold operator.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example 26
Source Project: flink   Source File: InstantiationUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
	if (original == null) {
		return null;
	}

	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		original.write(out);
	}

	final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {

		@SuppressWarnings("unchecked")
		T copy = (T) instantiate(original.getClass());
		copy.read(in);
		return copy;
	}
}
 
Example 27
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example 28
Source Project: flink   Source File: FromElementsFunction.java    License: Apache License 2.0 6 votes vote down vote up
public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
}
 
Example 29
Source Project: Flink-CEPplus   Source File: ImmutableListStateTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example 30
@Before
public void setUp() throws Exception {
	if (!aggrStateDesc.isSerializerInitialized()) {
		aggrStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
	}

	final String initValue = "42";

	ByteArrayOutputStream out = new ByteArrayOutputStream();
	aggrStateDesc.getSerializer().serialize(initValue, new DataOutputViewStreamWrapper(out));

	aggrState = ImmutableAggregatingState.createState(
			aggrStateDesc,
			out.toByteArray()
	);
}