org.apache.flink.core.memory.ByteArrayInputStreamWithPos Java Examples

The following examples show how to use org.apache.flink.core.memory.ByteArrayInputStreamWithPos. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SerializationProxiesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
Example #2
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadCompatibleVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2) {
		@Override
		public int[] getCompatibleVersions() {
			return new int[] {1, 2};
		}
	};
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
Example #3
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Verifies deserialization failure cases when reading a serializer from bytes, in the
 * case of a {@link InvalidClassException}.
 */
@Test
public void testSerializerSerializationWithInvalidClass() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in),
			new ArtificialCNFExceptionThrowingClassLoader(
				Thread.currentThread().getContextClassLoader(),
				Collections.singleton(IntSerializer.class.getName())),
			true);
	}
	Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer);
}
Example #4
Source File: TypeSerializerSerializationUtilTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Verifies deserialization failure cases when reading a serializer from bytes, in the
 * case of a {@link InvalidClassException}.
 */
@Test
public void testSerializerSerializationWithInvalidClass() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in),
			new ArtificialCNFExceptionThrowingClassLoader(
				Thread.currentThread().getContextClassLoader(),
				Collections.singleton(IntSerializer.class.getName())),
			true);
	}
	Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer);
}
Example #5
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
Example #6
Source File: PostVersionedIOReadableWritableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadVersioned() throws IOException {

	String payload = "test-data";
	TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		versionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(payload, restoredVersionedReadableWritable.getData());
}
Example #7
Source File: PostVersionedIOReadableWritableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadNonVersioned() throws IOException {
	int preVersionedPayload = 563;

	TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData());
}
Example #8
Source File: RocksDBKeySerializationUtilsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
Example #9
Source File: SerializationProxiesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
Example #10
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
Example #11
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadCompatibleVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2) {
		@Override
		public int[] getCompatibleVersions() {
			return new int[] {1, 2};
		}
	};
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
Example #12
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
Example #13
Source File: PostVersionedIOReadableWritableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadVersioned() throws IOException {

	String payload = "test-data";
	TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		versionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(payload, restoredVersionedReadableWritable.getData());
}
Example #14
Source File: PostVersionedIOReadableWritableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadNonVersioned() throws IOException {
	int preVersionedPayload = 563;

	TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData());
}
Example #15
Source File: RocksDBKeySerializationUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
Example #16
Source File: TypeSerializerSerializationUtilTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Verifies that reading and writing serializers work correctly.
 */
@Test
public void testSerializerSerialization() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(serializer, deserializedSerializer);
}
Example #17
Source File: RocksDBUtils.java    From bravo with Apache License 2.0 6 votes vote down vote up 
public static <V> List<V> deserializeList(
		byte[] valueBytes, TypeSerializer<V> elementSerializer, boolean ttl) throws IOException {
	if (valueBytes == null) {
		return null;
	}

	
	ByteArrayInputStreamWithPos is = new ByteArrayInputStreamWithPos(valueBytes);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(is);

	List<V> result = new ArrayList<>();
	V next;
	while ((next = deserializeNextElement(in, elementSerializer, ttl)) != null) {
		result.add(next);
	}
	return result;
}
Example #18
Source File: ValueStateValueReader.java    From bravo with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(KeyedStateRow row, Collector<V> out) throws Exception {
	if (!stateName.equals(row.getStateName())) {
		return;
	}

	byte[] valueBytes = row.getValueBytes();

	V value = null;
	try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) {
		DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs);
		skipTimestampIfTtlEnabled(iw);
		value = valueDeserializer.deserialize(iw);
	}
	if (value == null) {
		throw new RuntimeException("MapStates with null values are not supported at the moment.");
	} else {
		out.collect(value);
	}
}
Example #19
Source File: MapStateValueReader.java    From bravo with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(KeyedStateRow row, Collector<V> out) throws Exception {
	if (!stateName.equals(row.getStateName())) {
		return;
	}

	byte[] valueBytes = row.getValueBytes();

	V value = null;
	try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) {
		DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs);
		if (!iw.readBoolean()) {
			skipTimestampIfTtlEnabled(iw);
			value = valueDeserializer.deserialize(iw);
		}
	}
	if (value == null) {
		throw new RuntimeException("MapStates with null values are not supported at the moment.");
	} else {
		out.collect(value);
	}
}
Example #20
Source File: ListStateListReader.java    From bravo with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(KeyedStateRow row, Collector<Tuple2<K, List<V>>> out) throws Exception {
	if (!stateName.equals(row.getStateName())) {
		return;
	}

	byte[] keyAndNamespaceBytes = row.getKeyAndNamespaceBytes();
	byte[] valueBytes = row.getValueBytes();

	K key;
	try (ByteArrayInputStreamWithPos keyIs = new ByteArrayInputStreamWithPos(keyAndNamespaceBytes)) {
		DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(keyIs);
		iw.skipBytesToRead(keygroupPrefixBytes);
		key = RocksDBUtils.readKey(keyDeserializer, keyIs, iw, false);
	}

	out.collect(Tuple2.of(key, RocksDBUtils.deserializeList(valueBytes, valueDeserializer, ttlState)));
}
Example #21
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Verifies that reading and writing serializers work correctly.
 */
@Test
public void testSerializerSerialization() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(serializer, deserializedSerializer);
}
Example #22
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Verifies deserialization failure cases when reading a serializer from bytes, in the
 * case of a {@link InvalidClassException}.
 */
@Test
public void testSerializerSerializationWithInvalidClass() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in),
			new ArtificialCNFExceptionThrowingClassLoader(
				Thread.currentThread().getContextClassLoader(),
				Collections.singleton(IntSerializer.class.getName())),
			true);
	}
	Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer);
}
Example #23
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadSameVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(1);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
Example #24
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadCompatibleVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2) {
		@Override
		public int[] getCompatibleVersions() {
			return new int[] {1, 2};
		}
	};
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
Example #25
Source File: VersionedIOWriteableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMismatchVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2);
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
		Assert.fail("Version mismatch expected.");
	} catch (VersionMismatchException ignored) {

	}

	Assert.assertEquals(null, testWriteable.getData());
}
Example #26
Source File: PostVersionedIOReadableWritableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadVersioned() throws IOException {

	String payload = "test-data";
	TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		versionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(payload, restoredVersionedReadableWritable.getData());
}
Example #27
Source File: PostVersionedIOReadableWritableTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadNonVersioned() throws IOException {
	int preVersionedPayload = 563;

	TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload);

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
	try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		restoredVersionedReadableWritable.read(in);
	}

	Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData());
}
Example #28
Source File: CopyOnWriteSkipListStateMapTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <K, N, S> Map<N, Map<K, S>> readStateFromSnapshot(
	byte[] data,
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer) throws IOException {
	ByteArrayInputStreamWithPos inputStream = new ByteArrayInputStreamWithPos(data);
	DataInputView dataInputView = new DataInputViewStreamWrapper(inputStream);
	int size = dataInputView.readInt();

	Map<N, Map<K, S>> states = new HashMap<>();
	for (int i = 0; i < size; i++) {
		N namespace = namespaceSerializer.deserialize(dataInputView);
		K key = keySerializer.deserialize(dataInputView);
		S state = stateSerializer.deserialize(dataInputView);
		states.computeIfAbsent(namespace, (none) -> new HashMap<>()).put(key, state);
	}

	return states;
}
Example #29
Source File: RocksDBKeySerializationUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
Example #30
Source File: SerializationProxiesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}