Java Code Examples for org.apache.flink.core.memory.DataInputView

The following examples show how to use org.apache.flink.core.memory.DataInputView. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: InternalTimersSnapshotReaderWriters.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
protected void restoreKeyAndNamespaceSerializers(
		InternalTimersSnapshot<K, N> restoredTimersSnapshot,
		DataInputView in) throws IOException {

	DataInputViewStream dis = new DataInputViewStream(in);
	try {
		final TypeSerializer<K> keySerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);
		final TypeSerializer<N> namespaceSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);

		restoredTimersSnapshot.setKeySerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(keySerializer));
		restoredTimersSnapshot.setNamespaceSerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(namespaceSerializer));
	} catch (ClassNotFoundException exception) {
		throw new IOException(exception);
	}
}
 
Example 2
@Override
protected void read(DataInputView in, boolean wasVersioned) throws IOException {
	int noOfTimerServices = in.readInt();

	for (int i = 0; i < noOfTimerServices; i++) {
		String serviceName = in.readUTF();

		int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
		InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
			.getReaderForVersion(readerVersion, userCodeClassLoader)
			.readTimersSnapshot(in);

		InternalTimerServiceImpl<K, ?> timerService = registerOrGetTimerService(
			serviceName,
			restoredTimersSnapshot);

		timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
	}
}
 
Example 3
Source Project: flink   Source File: OperatorStateRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 
Example 4
Source Project: flink   Source File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.decoder.setIn(source);
		return this.reader.read(reuse, this.decoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 
Example 5
Source Project: flink   Source File: OperatorStateRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
 
Example 6
Source Project: flink   Source File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example 7
/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 
Example 8
Source Project: flink   Source File: MapSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);

	for (int i = 0; i < size; ++i) {
		keySerializer.copy(source, target);
		
		boolean isNull = source.readBoolean();
		target.writeBoolean(isNull);
		
		if (!isNull) {
			valueSerializer.copy(source, target);
		}
	}
}
 
Example 9
Source Project: flink   Source File: PojoSerializerSnapshotData.java    License: Apache License 2.0 5 votes vote down vote up
private static BiFunctionWithException<DataInputView, String, TypeSerializerSnapshot<?>, IOException> snapshotReader(ClassLoader cl) {
	return (input, unused) -> {
		try {
			return TypeSerializerSnapshot.readVersionedSnapshot(input, cl);
		}
		catch (Throwable t) {
			LOG.warn("Exception while reading serializer snapshot.", t);
			return null;
		}
	};
}
 
Example 10
@Override
public final void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	if (readVersion != ADAPTER_VERSION) {
		throw new IOException("Wrong/unexpected version for the TypeSerializerConfigSnapshot: " + readVersion);
	}

	serializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true);

	// now delegate to the snapshots own reading code
	setUserCodeClassLoader(userCodeClassLoader);
	read(in);
}
 
Example 11
Source Project: flink   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(
	DataInputView source, DataOutputView target) throws IOException {
	boolean hasTransactionalId = source.readBoolean();
	target.writeBoolean(hasTransactionalId);
	if (hasTransactionalId) {
		target.writeUTF(source.readUTF());
	}
	target.writeLong(source.readLong());
	target.writeShort(source.readShort());
}
 
Example 12
Source Project: flink   Source File: TupleComparatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	if (deserializedFields1 == null) {
		instantiateDeserializationUtils();
	}
	
	int i = 0;
	try {
		for (; i < serializers.length; i++) {
			deserializedFields1[i] = serializers[i].deserialize(deserializedFields1[i], firstSource);
			deserializedFields2[i] = serializers[i].deserialize(deserializedFields2[i], secondSource);
		}
		
		for (i = 0; i < keyPositions.length; i++) {
			int keyPos = keyPositions[i];
			int cmp = comparators[i].compare(deserializedFields1[keyPos], deserializedFields2[keyPos]);
			if (cmp != 0) {
				return cmp;
			}
		}
		
		return 0;
	} catch (NullPointerException npex) {
		throw new NullKeyFieldException(keyPositions[i]);
	} catch (IndexOutOfBoundsException iobex) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i], iobex);
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: FloatValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readFloat();
	}
}
 
Example 14
Source Project: flink   Source File: IntPrimitiveArraySerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int[] deserialize(DataInputView source) throws IOException {
	final int len = source.readInt();
	int[] result = new int[len];
	
	for (int i = 0; i < len; i++) {
		result[i] = source.readInt();
	}
	
	return result;
}
 
Example 15
Source Project: flink   Source File: RowSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int len = fieldSerializers.length;

	// copy bitmask
	readIntoAndCopyMask(source, target, mask);

	// copy non-null fields
	for (int fieldPos = 0; fieldPos < len; fieldPos++) {
		if (!mask[legacyOffset + fieldPos]) {
			fieldSerializers[fieldPos].copy(source, target);
		}
	}
}
 
Example 16
Source Project: flink   Source File: StringValue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
	int len = in.readUnsignedByte();
	target.writeByte(len);

	if (len >= HIGH_BIT) {
		int shift = 7;
		int curr;
		len = len & 0x7f;
		while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
			len |= (curr & 0x7f) << shift;
			shift += 7;
			target.writeByte(curr);
		}
		len |= curr << shift;
		target.writeByte(curr);
	}

	for (int i = 0; i < len; i++) {
		int c = in.readUnsignedByte();
		target.writeByte(c);
		while (c >= HIGH_BIT) {
			c = in.readUnsignedByte();
			target.writeByte(c);
		}
	}
}
 
Example 17
Source Project: flink   Source File: CustomIntSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Integer deserialize(DataInputView source) throws IOException {
	int header = source.readInt();
	if (header != MAGIC_VALUE) {
		throw new RuntimeException(String.format("Invalid magic value, expected %d found %d", MAGIC_VALUE, header));
	}
	return source.readInt();
}
 
Example 18
Source Project: flink   Source File: LinkedOptionalMapSerializer.java    License: Apache License 2.0 5 votes vote down vote up
public static <K, V> LinkedOptionalMap<K, V> readOptionalMap(
	DataInputView in,
	BiFunctionWithException<DataInputView, String, K, IOException> keyReader,
	BiFunctionWithException<DataInputView, String, V, IOException> valueReader) throws IOException {

	final long header = in.readLong();
	checkState(header == HEADER, "Corrupted stream received header %s", header);

	long mapSize = in.readInt();
	LinkedOptionalMap<K, V> map = new LinkedOptionalMap<>();
	for (int i = 0; i < mapSize; i++) {
		String keyName = in.readUTF();

		final K key;
		if (in.readBoolean()) {
			key = tryReadFrame(in, keyName, keyReader);
		}
		else {
			key = null;
		}

		final V value;
		if (in.readBoolean()) {
			value = tryReadFrame(in, keyName, valueReader);
		}
		else {
			value = null;
		}

		map.put(keyName, key, value);
	}
	return map;
}
 
Example 19
Source Project: flink   Source File: IntComparator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	int i1 = firstSource.readInt();
	int i2 = secondSource.readInt();
	int comp = (i1 < i2 ? -1 : (i1 == i2 ? 0 : 1)); 
	return ascendingComparison ? comp : -comp; 
}
 
Example 20
Source Project: Flink-CEPplus   Source File: CollectionInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ElementType deserialize(ElementType reuse, DataInputView source) throws IOException {
	if (failOnRead) {
		throw new TestException();
	}
	return new ElementType(source.readInt());
}
 
Example 21
Source Project: flink   Source File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readByte();
	}
}
 
Example 22
Source Project: flink   Source File: ArrayDataSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		DataInputViewStream inStream = new DataInputViewStream(in);
		this.previousType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousEleSer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
}
 
Example 23
private void assertSerializerIsAbleToReadOldData(TypeSerializer<ElementT> serializer) throws IOException {
	DataInputView input = dataUnderTest();

	final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
	for (int i = 0; i < testSpecification.testDataCount; i++) {
		final ElementT result = serializer.deserialize(input);
		assertThat(result, matcher);
	}
}
 
Example 24
Source Project: flink   Source File: TupleSerializerConfigSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}
 
Example 25
Source Project: flink   Source File: Kafka09ITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String deserialize(byte[] messageKey, byte[] message, String topic, int partition,
	long offset) throws IOException {
	cnt++;
	DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message));
	String e = ser.deserialize(in);
	return e;
}
 
Example 26
Source Project: flink   Source File: NullableSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private boolean deserializeNull(DataInputView source) throws IOException {
	boolean isNull = source.readBoolean();
	if (isNull) {
		source.skipBytesToRead(padding.length);
	}
	return isNull;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FlinkKafkaProducer.KafkaTransactionState deserialize(
	FlinkKafkaProducer.KafkaTransactionState reuse,
	DataInputView source) throws IOException {
	return deserialize(source);
}
 
Example 28
Source Project: flink   Source File: StringValueArray.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	copyInternal(source, target);
}
 
Example 29
Source Project: Flink-CEPplus   Source File: NullValueComparator.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public NullValue readWithKeyDenormalization(NullValue reuse, DataInputView source) throws IOException {
	throw new UnsupportedOperationException();
}
 
Example 30
Source Project: flink   Source File: BooleanType.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	this.value = in.readBoolean();
}