org.apache.flink.core.memory.DataOutputViewStreamWrapper Java Examples

The following examples show how to use org.apache.flink.core.memory.DataOutputViewStreamWrapper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that reading and writing serializers work correctly.
 */
@Test
public void testSerializerSerialization() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(serializer, deserializedSerializer);
}
 
Example #2
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
 
Example #3
Source File: FailingCollectionSource.java    From flink with Apache License 2.0 6 votes vote down vote up
public FailingCollectionSource(
	TypeSerializer<T> serializer,
	Iterable<T> elements,
	int failureAfterNumElements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
	checkArgument(failureAfterNumElements > 0);
	this.failureAfterNumElements = failureAfterNumElements;
	this.checkpointedEmittedNums = new HashMap<>();
}
 
Example #4
Source File: InstantiationUtil.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
	if (original == null) {
		return null;
	}

	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		original.write(out);
	}

	final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {

		@SuppressWarnings("unchecked")
		T copy = (T) instantiate(original.getClass());
		copy.read(in);
		return copy;
	}
}
 
Example #5
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
 
Example #6
Source File: SerializationProxiesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example #7
Source File: FromElementsFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
}
 
Example #8
Source File: ImmutableListStateTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example #9
Source File: TypeSerializerSerializationUtilTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example #10
Source File: FoldApplyProcessAllWindowFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example #11
Source File: RocksDBKeySerializationUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example #12
Source File: InternalPriorityQueueTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public int compare(TestElement o1, TestElement o2) {

	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os);
	try {
		TestElementSerializer.INSTANCE.serialize(o1, ow);
		byte[] a1 = os.toByteArray();
		os.reset();
		TestElementSerializer.INSTANCE.serialize(o2, ow);
		byte[] a2 = os.toByteArray();
		return UnsignedBytes.lexicographicalComparator().compare(a1, a2);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
Example #13
Source File: FoldApplyWindowFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	// out type is not used, just use this for the execution config
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example #14
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
 
Example #15
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
 
Example #16
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example #17
Source File: ImmutableListStateTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example #18
Source File: RocksDBKeySerializationUtilsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example #19
Source File: CopyOnWriteStateTableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * This tests that resource can be released for a successful snapshot.
 */
@Test
public void testReleaseForSuccessfulSnapshot() throws IOException {
	int numberOfKeyGroups = 10;
	CopyOnWriteStateTable<Integer, Integer, Float> table = createStateTableForSnapshotRelease(numberOfKeyGroups);

	ByteArrayOutputStreamWithPos byteArrayOutputStreamWithPos = new ByteArrayOutputStreamWithPos();
	DataOutputView dataOutputView = new DataOutputViewStreamWrapper(byteArrayOutputStreamWithPos);

	CopyOnWriteStateTableSnapshot<Integer, Integer, Float> snapshot = table.stateSnapshot();
	for (int group = 0; group < numberOfKeyGroups; group++) {
		snapshot.writeStateInKeyGroup(dataOutputView, group);
		// resource used by one key group should be released after the snapshot is successful
		Assert.assertTrue(isResourceReleasedForKeyGroup(table, group));
	}
	snapshot.release();
	verifyResourceIsReleasedForAllKeyGroup(table, 1);
}
 
Example #20
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example #21
Source File: EventWithAggregatorsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example #22
Source File: EventWithAggregatorsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example #23
Source File: KeyedStateCheckpointOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadWriteMissingKeyGroups() throws Exception {
	final KeyGroupRange keyRange = new KeyGroupRange(0, 2);
	KeyedStateCheckpointOutputStream stream = createStream(keyRange);

	DataOutputView dov = new DataOutputViewStreamWrapper(stream);
	stream.startNewKeyGroup(1);
	dov.writeInt(1);

	KeyGroupsStateHandle fullHandle = stream.closeAndGetHandle();

	int count = 0;
	try (FSDataInputStream in = fullHandle.openInputStream()) {
		DataInputView div = new DataInputViewStreamWrapper(in);
		for (int kg : fullHandle.getKeyGroupRange()) {
			long off = fullHandle.getOffsetForKeyGroup(kg);
			if (off >= 0) {
				in.seek(off);
				Assert.assertEquals(1, div.readInt());
				++count;
			}
		}
	}

	Assert.assertEquals(1, count);
}
 
Example #24
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 
Example #25
Source File: StreamGroupedFold.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	outTypeSerializer = outTypeInfo.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		outTypeSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
				initialValue.getClass().getSimpleName() + " of fold operator.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example #26
Source File: InstantiationUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
	if (original == null) {
		return null;
	}

	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		original.write(out);
	}

	final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {

		@SuppressWarnings("unchecked")
		T copy = (T) instantiate(original.getClass());
		copy.read(in);
		return copy;
	}
}
 
Example #27
Source File: FoldApplyProcessAllWindowFunction.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void setOutputType(TypeInformation<R> outTypeInfo, ExecutionConfig executionConfig) {
	accSerializer = accTypeInformation.createSerializer(executionConfig);

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);

	try {
		accSerializer.serialize(initialValue, out);
	} catch (IOException ioe) {
		throw new RuntimeException("Unable to serialize initial value of type " +
			initialValue.getClass().getSimpleName() + " of fold window function.", ioe);
	}

	serializedInitialValue = baos.toByteArray();
}
 
Example #28
Source File: FromElementsFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
public FromElementsFunction(TypeSerializer<T> serializer, Iterable<T> elements) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);

	int count = 0;
	try {
		for (T element : elements) {
			serializer.serialize(element, wrapper);
			count++;
		}
	}
	catch (Exception e) {
		throw new IOException("Serializing the source elements failed: " + e.getMessage(), e);
	}

	this.serializer = serializer;
	this.elementsSerialized = baos.toByteArray();
	this.numElements = count;
}
 
Example #29
Source File: ImmutableListStateTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example #30
Source File: ImmutableAggregatingStateTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	if (!aggrStateDesc.isSerializerInitialized()) {
		aggrStateDesc.initializeSerializerUnlessSet(new ExecutionConfig());
	}

	final String initValue = "42";

	ByteArrayOutputStream out = new ByteArrayOutputStream();
	aggrStateDesc.getSerializer().serialize(initValue, new DataOutputViewStreamWrapper(out));

	aggrState = ImmutableAggregatingState.createState(
			aggrStateDesc,
			out.toByteArray()
	);
}