Java Code Examples for org.apache.flink.core.memory.DataInputDeserializer

The following examples show how to use org.apache.flink.core.memory.DataInputDeserializer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private <K, N, U> void assertKeyGroupKeyNamespaceUserKeyBytes(
	K key,
	int keyGroup,
	int prefixBytes,
	TypeSerializer<K> keySerializer,
	N namespace,
	TypeSerializer<N> namespaceSerializer,
	U userKey,
	TypeSerializer<U> userKeySerializer,
	DataInputDeserializer deserializer,
	boolean ambiguousCompositeKeyPossible) throws IOException {
	assertKeyGroupKeyNamespaceBytes(
		key,
		keyGroup,
		prefixBytes,
		keySerializer,
		namespace,
		namespaceSerializer,
		deserializer,
		ambiguousCompositeKeyPossible);
	Assert.assertEquals(userKey, userKeySerializer.deserialize(deserializer));
}
 
Example 2
RocksDBPriorityQueueSetFactory(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfKeyGroups,
	Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDB db,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.kvStateInformation = kvStateInformation;
	this.db = db;
	this.writeBatchWrapper = writeBatchWrapper;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.sharedElementOutView = new DataOutputSerializer(128);
	this.sharedElementInView = new DataInputDeserializer();
}
 
Example 3
Source Project: flink   Source File: RocksDBKeySerializationUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNamespaceSerializationAndDeserialization() throws Exception {
	final DataOutputSerializer outputView = new DataOutputSerializer(8);
	final DataInputDeserializer inputView = new DataInputDeserializer();

	for (int orgNamespace = 0; orgNamespace < 100; ++orgNamespace) {
		outputView.clear();
		RocksDBKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, false);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		int deserializedNamepsace = RocksDBKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, false);
		Assert.assertEquals(orgNamespace, deserializedNamepsace);

		RocksDBKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, true);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		deserializedNamepsace = RocksDBKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, true);
		Assert.assertEquals(orgNamespace, deserializedNamepsace);
	}
}
 
Example 4
Source Project: flink   Source File: RocksStateKeysIterator.java    License: Apache License 2.0 6 votes vote down vote up
public RocksStateKeysIterator(
	@Nonnull RocksIteratorWrapper iterator,
	@Nonnull String state,
	@Nonnull TypeSerializer<K> keySerializer,
	int keyGroupPrefixBytes,
	boolean ambiguousKeyPossible,
	@Nonnull byte[] namespaceBytes) {
	this.iterator = iterator;
	this.state = state;
	this.keySerializer = keySerializer;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.namespaceBytes = namespaceBytes;
	this.nextKey = null;
	this.previousKey = null;
	this.ambiguousKeyPossible = ambiguousKeyPossible;
	this.byteArrayDataInputView = new DataInputDeserializer();
}
 
Example 5
Source Project: flink   Source File: AbstractRocksDBState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new RocksDB backed state.
 *
 * @param columnFamily The RocksDB column family that this state is associated to.
 * @param namespaceSerializer The serializer for the namespace.
 * @param valueSerializer The serializer for the state.
 * @param defaultValue The default value for the state.
 * @param backend The backend for which this state is bind to.
 */
protected AbstractRocksDBState(
		ColumnFamilyHandle columnFamily,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer,
		V defaultValue,
		RocksDBKeyedStateBackend<K> backend) {

	this.namespaceSerializer = namespaceSerializer;
	this.backend = backend;

	this.columnFamily = columnFamily;

	this.writeOptions = backend.getWriteOptions();
	this.valueSerializer = Preconditions.checkNotNull(valueSerializer, "State value serializer");
	this.defaultValue = defaultValue;

	this.dataOutputView = new DataOutputSerializer(128);
	this.dataInputView = new DataInputDeserializer();
	this.sharedKeyNamespaceSerializer = backend.getSharedRocksKeyBuilder();
}
 
Example 6
@Test
public void testNamespaceSerializationAndDeserialization() throws Exception {
	final DataOutputSerializer outputView = new DataOutputSerializer(8);
	final DataInputDeserializer inputView = new DataInputDeserializer();

	for (int orgNamespace = 0; orgNamespace < 100; ++orgNamespace) {
		outputView.clear();
		RocksDBKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, false);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		int deserializedNamepsace = RocksDBKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, false);
		Assert.assertEquals(orgNamespace, deserializedNamepsace);

		RocksDBKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, true);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		deserializedNamepsace = RocksDBKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, true);
		Assert.assertEquals(orgNamespace, deserializedNamepsace);
	}
}
 
Example 7
Source Project: flink   Source File: RocksDBMapState.java    License: Apache License 2.0 6 votes vote down vote up
RocksDBMapEntry(
		@Nonnull final RocksDB db,
		@Nonnegative final int userKeyOffset,
		@Nonnull final byte[] rawKeyBytes,
		@Nonnull final byte[] rawValueBytes,
		@Nonnull final TypeSerializer<UK> keySerializer,
		@Nonnull final TypeSerializer<UV> valueSerializer,
		@Nonnull DataInputDeserializer dataInputView) {
	this.db = db;

	this.userKeyOffset = userKeyOffset;
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;

	this.rawKeyBytes = rawKeyBytes;
	this.rawValueBytes = rawValueBytes;
	this.deleted = false;
	this.dataInputView = dataInputView;
}
 
Example 8
Source Project: Flink-CEPplus   Source File: StreamElementSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T, X extends StreamElement> X serializeAndDeserialize(
		X record,
		StreamElementSerializer<T> serializer) throws IOException {

	DataOutputSerializer output = new DataOutputSerializer(32);
	serializer.serialize(record, output);

	// additional binary copy step
	DataInputDeserializer copyInput = new DataInputDeserializer(output.getByteArray(), 0, output.length());
	DataOutputSerializer copyOutput = new DataOutputSerializer(32);
	serializer.copy(copyInput, copyOutput);

	DataInputDeserializer input = new DataInputDeserializer(copyOutput.getByteArray(), 0, copyOutput.length());
	return (X) serializer.deserialize(input);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: KvStateSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deserializes the value with the given serializer.
 *
 * @param serializedValue Serialized value of type T
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized value or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> T deserializeValue(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue == null) {
		return null;
	} else {
		final DataInputDeserializer deser = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);
		final T value = serializer.deserialize(deser);
		if (deser.available() > 0) {
			throw new IOException(
				"Unconsumed bytes in the deserialized value. " +
					"This indicates a mismatch in the value serializers " +
					"used by the KvState instance and this access.");
		}
		return value;
	}
}
 
Example 10
Source Project: flink   Source File: StreamElementSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T, X extends StreamElement> X serializeAndDeserialize(
		X record,
		StreamElementSerializer<T> serializer) throws IOException {

	DataOutputSerializer output = new DataOutputSerializer(32);
	serializer.serialize(record, output);

	// additional binary copy step
	DataInputDeserializer copyInput = new DataInputDeserializer(output.getByteArray(), 0, output.length());
	DataOutputSerializer copyOutput = new DataOutputSerializer(32);
	serializer.copy(copyInput, copyOutput);

	DataInputDeserializer input = new DataInputDeserializer(copyOutput.getByteArray(), 0, copyOutput.length());
	return (X) serializer.deserialize(input);
}
 
Example 11
private KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<
	TestElement, RocksDBCachingPriorityQueueSet<TestElement>> newFactory() {

	return (keyGroupId, numKeyGroups, keyExtractorFunction, elementComparator) -> {
		DataOutputSerializer outputStreamWithPos = new DataOutputSerializer(128);
		DataInputDeserializer inputStreamWithPos = new DataInputDeserializer();
		int keyGroupPrefixBytes = RocksDBKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(numKeyGroups);
		TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(32);
		return new RocksDBCachingPriorityQueueSet<>(
			keyGroupId,
			keyGroupPrefixBytes,
			rocksDBResource.getRocksDB(),
			rocksDBResource.getReadOptions(),
			rocksDBResource.getDefaultColumnFamily(),
			TestElementSerializer.INSTANCE,
			outputStreamWithPos,
			inputStreamWithPos,
			rocksDBResource.getBatchWrapper(),
			orderedSetCache);
	};
}
 
Example 12
Source Project: flink   Source File: KvStateSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deserializes all kv pairs with the given serializer.
 *
 * @param serializedValue Serialized value of type Map&lt;UK, UV&gt;
 * @param keySerializer   Serializer for UK
 * @param valueSerializer Serializer for UV
 * @param <UK>            Type of the key
 * @param <UV>            Type of the value.
 * @return Deserialized map or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <UK, UV> Map<UK, UV> deserializeMap(byte[] serializedValue, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
	if (serializedValue != null) {
		DataInputDeserializer in = new DataInputDeserializer(serializedValue, 0, serializedValue.length);

		Map<UK, UV> result = new HashMap<>();
		while (in.available() > 0) {
			UK key = keySerializer.deserialize(in);

			boolean isNull = in.readBoolean();
			UV value = isNull ? null : valueSerializer.deserialize(in);

			result.put(key, value);
		}

		return result;
	} else {
		return null;
	}
}
 
Example 13
Source Project: flink   Source File: RocksDBPriorityQueueSetFactory.java    License: Apache License 2.0 6 votes vote down vote up
RocksDBPriorityQueueSetFactory(
	KeyGroupRange keyGroupRange,
	int keyGroupPrefixBytes,
	int numberOfKeyGroups,
	Map<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	RocksDB db,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory) {
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.kvStateInformation = kvStateInformation;
	this.db = db;
	this.writeBatchWrapper = writeBatchWrapper;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.columnFamilyOptionsFactory = columnFamilyOptionsFactory;
	this.sharedElementOutView = new DataOutputSerializer(128);
	this.sharedElementInView = new DataInputDeserializer();
}
 
Example 14
Source Project: flink   Source File: RocksDBMapState.java    License: Apache License 2.0 6 votes vote down vote up
RocksDBMapEntry(
		@Nonnull final RocksDB db,
		@Nonnegative final int userKeyOffset,
		@Nonnull final byte[] rawKeyBytes,
		@Nonnull final byte[] rawValueBytes,
		@Nonnull final TypeSerializer<UK> keySerializer,
		@Nonnull final TypeSerializer<UV> valueSerializer,
		@Nonnull DataInputDeserializer dataInputView) {
	this.db = db;

	this.userKeyOffset = userKeyOffset;
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;

	this.rawKeyBytes = rawKeyBytes;
	this.rawValueBytes = rawValueBytes;
	this.deleted = false;
	this.dataInputView = dataInputView;
}
 
Example 15
Source Project: flink   Source File: RocksDBKeySerializationUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeySerializationAndDeserialization() throws Exception {
	final DataOutputSerializer outputView = new DataOutputSerializer(8);
	final DataInputDeserializer inputView = new DataInputDeserializer();

	// test for key
	for (int orgKey = 0; orgKey < 100; ++orgKey) {
		outputView.clear();
		RocksDBKeySerializationUtils.writeKey(orgKey, IntSerializer.INSTANCE, outputView, false);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		int deserializedKey = RocksDBKeySerializationUtils.readKey(IntSerializer.INSTANCE, inputView, false);
		Assert.assertEquals(orgKey, deserializedKey);

		RocksDBKeySerializationUtils.writeKey(orgKey, IntSerializer.INSTANCE, outputView, true);
		inputView.setBuffer(outputView.getCopyOfBuffer());
		deserializedKey = RocksDBKeySerializationUtils.readKey(IntSerializer.INSTANCE, inputView, true);
		Assert.assertEquals(orgKey, deserializedKey);
	}
}
 
Example 16
Source Project: flink   Source File: RocksDBCachingPriorityQueueSet.java    License: Apache License 2.0 6 votes vote down vote up
RocksDBCachingPriorityQueueSet(
	@Nonnegative int keyGroupId,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull RocksDB db,
	@Nonnull ReadOptions readOptions,
	@Nonnull ColumnFamilyHandle columnFamilyHandle,
	@Nonnull TypeSerializer<E> byteOrderProducingSerializer,
	@Nonnull DataOutputSerializer outputStream,
	@Nonnull DataInputDeserializer inputStream,
	@Nonnull RocksDBWriteBatchWrapper batchWrapper,
	@Nonnull OrderedByteArraySetCache orderedByteArraySetCache) {
	this.db = db;
	this.readOptions = readOptions;
	this.columnFamilyHandle = columnFamilyHandle;
	this.byteOrderProducingSerializer = byteOrderProducingSerializer;
	this.batchWrapper = batchWrapper;
	this.outputView = outputStream;
	this.inputView = inputStream;
	this.orderedCache = orderedByteArraySetCache;
	this.allElementsInCache = false;
	this.groupPrefixBytes = createKeyGroupBytes(keyGroupId, keyGroupPrefixBytes);
	this.seekHint = groupPrefixBytes;
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 
Example 17
Source Project: flink   Source File: KvStateSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deserializes the value with the given serializer.
 *
 * @param serializedValue Serialized value of type T
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized value or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> T deserializeValue(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue == null) {
		return null;
	} else {
		final DataInputDeserializer deser = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);
		final T value = serializer.deserialize(deser);
		if (deser.available() > 0) {
			throw new IOException(
				"Unconsumed bytes in the deserialized value. " +
					"This indicates a mismatch in the value serializers " +
					"used by the KvState instance and this access.");
		}
		return value;
	}
}
 
Example 18
Source Project: flink   Source File: AbstractRocksDBState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new RocksDB backed state.
 *
 * @param columnFamily The RocksDB column family that this state is associated to.
 * @param namespaceSerializer The serializer for the namespace.
 * @param valueSerializer The serializer for the state.
 * @param defaultValue The default value for the state.
 * @param backend The backend for which this state is bind to.
 */
protected AbstractRocksDBState(
		ColumnFamilyHandle columnFamily,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<V> valueSerializer,
		V defaultValue,
		RocksDBKeyedStateBackend<K> backend) {

	this.namespaceSerializer = namespaceSerializer;
	this.backend = backend;

	this.columnFamily = columnFamily;

	this.writeOptions = backend.getWriteOptions();
	this.valueSerializer = Preconditions.checkNotNull(valueSerializer, "State value serializer");
	this.defaultValue = defaultValue;

	this.dataOutputView = new DataOutputSerializer(128);
	this.dataInputView = new DataInputDeserializer();
	this.sharedKeyNamespaceSerializer = backend.getSharedRocksKeyBuilder();
}
 
Example 19
Source Project: stateful-functions   Source File: ProtobufSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void deserializeCopiedMessage() throws IOException {
  DataInputDeserializer in = serialize(SimpleMessage.class, originalMessage);

  ProtobufSerializer<SimpleMessage> serializer =
      ProtobufSerializer.forMessageGeneratedClass(SimpleMessage.class);
  DataOutputSerializer out = new DataOutputSerializer(32);

  serializer.copy(in, out);
  SimpleMessage message = deserialize(SimpleMessage.class, out);

  assertThat(message, is(originalMessage));
}
 
Example 20
Source Project: Flink-CEPplus   Source File: LinkedOptionalMapSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
private static <T> T tryReadFrame(DataInputView in, String keyName, BiFunctionWithException<DataInputView, String, T, IOException> reader) throws IOException {
	final int bufferSize = in.readInt();
	final byte[] buffer = new byte[bufferSize];
	in.readFully(buffer);
	DataInputDeserializer frame = new DataInputDeserializer(buffer);
	return reader.apply(frame, keyName);
}
 
Example 21
Source Project: Flink-CEPplus   Source File: KryoSerializerSnapshotTest.java    License: Apache License 2.0 5 votes vote down vote up
private static TypeSerializerSnapshot<Animal> kryoSnapshotWithMissingClass() throws IOException {
	DataInputView in = new DataInputDeserializer(unLoadableSnapshotBytes());

	return TypeSerializerSnapshot.readVersionedSnapshot(
		in,
		KryoSerializerSnapshotTest.class.getClassLoader());
}
 
Example 22
Source Project: stateful-functions   Source File: ProtobufSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@SafeVarargs
private static <M extends Message> DataInputDeserializer serialize(Class<M> type, M... items)
    throws IOException {
  ProtobufSerializer<M> serializer = ProtobufSerializer.forMessageGeneratedClass(type);

  DataOutputSerializer out = new DataOutputSerializer(512);
  for (Object message : items) {
    serializer.serialize(type.cast(message), out);
  }
  return new DataInputDeserializer(out.getCopyOfBuffer());
}
 
Example 23
private static DataInputView contentsOf(Path path) {
	try {
		byte[] bytes = Files.readAllBytes(path);
		return new DataInputDeserializer(bytes);
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to read " + path, e);
	}
}
 
Example 24
Source Project: flink-statefun   Source File: ProtobufSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void deserializeCopiedMessage() throws IOException {
  DataInputDeserializer in = serialize(SimpleMessage.class, originalMessage);

  ProtobufSerializer<SimpleMessage> serializer =
      ProtobufSerializer.forMessageGeneratedClass(SimpleMessage.class);
  DataOutputSerializer out = new DataOutputSerializer(32);

  serializer.copy(in, out);
  SimpleMessage message = deserialize(SimpleMessage.class, out);

  assertThat(message, is(originalMessage));
}
 
Example 25
@Test
public void testRestoreCompositeTypeSerializer() throws IOException {
	// the target compatibilities of the nested serializers doesn't matter,
	// because we're only testing the restore serializer
	TypeSerializer<?>[] testNestedSerializers = {
		new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS),
		new NestedSerializer(TargetCompatibility.INCOMPATIBLE),
		new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION)
	};

	TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer("outer-config", testNestedSerializers);

	TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration();

	DataOutputSerializer out = new DataOutputSerializer(128);
	TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot);

	DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
	testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot(
		in, Thread.currentThread().getContextClassLoader());

	// now, restore the composite type serializer;
	// the restored nested serializer should be a RestoredNestedSerializer
	testSerializer = (TestCompositeTypeSerializer) testSerializerSnapshot.restoreSerializer();
	Assert.assertTrue(testSerializer.getNestedSerializers()[0].getClass() == RestoredNestedSerializer.class);
	Assert.assertTrue(testSerializer.getNestedSerializers()[1].getClass() == RestoredNestedSerializer.class);
	Assert.assertTrue(testSerializer.getNestedSerializers()[2].getClass() == RestoredNestedSerializer.class);
}
 
Example 26
private <K, N> void testSetKeyNamespaceInternal(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	Collection<K> testKeys,
	Collection<N> testNamespaces,
	int maxParallelism) throws IOException {
	final int prefixBytes = maxParallelism > Byte.MAX_VALUE ? 2 : 1;

	RocksDBSerializedCompositeKeyBuilder<K> keyBuilder =
		createRocksDBSerializedCompositeKeyBuilder(keySerializer, prefixBytes);

	final DataInputDeserializer deserializer = new DataInputDeserializer();

	final boolean ambiguousPossible = keyBuilder.isAmbiguousCompositeKeyPossible(namespaceSerializer);

	for (K testKey : testKeys) {
		int keyGroup = setKeyAndReturnKeyGroup(keyBuilder, testKey, maxParallelism);
		for (N testNamespace : testNamespaces) {
			byte[] compositeBytes = keyBuilder.buildCompositeKeyNamespace(testNamespace, namespaceSerializer);
			deserializer.setBuffer(compositeBytes);
			assertKeyGroupKeyNamespaceBytes(
				testKey,
				keyGroup,
				prefixBytes,
				keySerializer,
				testNamespace,
				namespaceSerializer,
				deserializer,
				ambiguousPossible);
			Assert.assertEquals(0, deserializer.available());
		}
	}
}
 
Example 27
Source Project: flink   Source File: TypeSerializerUpgradeTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> TypeSerializerSnapshot<T> writeAndThenReadSerializerSnapshot(
		TypeSerializer<T> serializer) throws IOException {

	DataOutputSerializer out = new DataOutputSerializer(INITIAL_OUTPUT_BUFFER_SIZE);
	writeSerializerSnapshotCurrentFormat(out, serializer);

	DataInputDeserializer in = new DataInputDeserializer(out.wrapAsByteBuffer());
	return readSerializerSnapshotCurrentFormat(
			in,
			Thread.currentThread().getContextClassLoader());
}
 
Example 28
Source Project: Flink-CEPplus   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
private static <UK> UK deserializeUserKey(
	DataInputDeserializer dataInputView,
	int userKeyOffset,
	byte[] rawKeyBytes,
	TypeSerializer<UK> keySerializer) throws IOException {
	dataInputView.setBuffer(rawKeyBytes, userKeyOffset, rawKeyBytes.length - userKeyOffset);
	return keySerializer.deserialize(dataInputView);
}
 
Example 29
Source Project: Flink-CEPplus   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
private static <UV> UV deserializeUserValue(
	DataInputDeserializer dataInputView,
	byte[] rawValueBytes,
	TypeSerializer<UV> valueSerializer) throws IOException {

	dataInputView.setBuffer(rawValueBytes);

	boolean isNull = dataInputView.readBoolean();

	return isNull ? null : valueSerializer.deserialize(dataInputView);
}
 
Example 30
Source Project: Flink-CEPplus   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
RocksDBMapIterator(
	final RocksDB db,
	final byte[] keyPrefixBytes,
	final TypeSerializer<UK> keySerializer,
	final TypeSerializer<UV> valueSerializer,
	DataInputDeserializer dataInputView) {

	this.db = db;
	this.keyPrefixBytes = keyPrefixBytes;
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;
	this.dataInputView = dataInputView;
}