org.apache.flink.core.memory.DataInputView Java Examples

The following examples show how to use org.apache.flink.core.memory.DataInputView. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: apache   File: InternalTimersSnapshotReaderWriters.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
protected void restoreKeyAndNamespaceSerializers(
		InternalTimersSnapshot<K, N> restoredTimersSnapshot,
		DataInputView in) throws IOException {

	DataInputViewStream dis = new DataInputViewStream(in);
	try {
		final TypeSerializer<K> keySerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);
		final TypeSerializer<N> namespaceSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);

		restoredTimersSnapshot.setKeySerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(keySerializer));
		restoredTimersSnapshot.setNamespaceSerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(namespaceSerializer));
	} catch (ClassNotFoundException exception) {
		throw new IOException(exception);
	}
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: InternalTimerServiceSerializationProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void read(DataInputView in, boolean wasVersioned) throws IOException {
	int noOfTimerServices = in.readInt();

	for (int i = 0; i < noOfTimerServices; i++) {
		String serviceName = in.readUTF();

		int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
		InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
			.getReaderForVersion(readerVersion, userCodeClassLoader)
			.readTimersSnapshot(in);

		InternalTimerServiceImpl<K, ?> timerService = registerOrGetTimerService(
			serviceName,
			restoredTimersSnapshot);

		timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
	}
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: OperatorStateRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 
Example #4
Source Project: flink   Author: apache   File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.decoder.setIn(source);
		return this.reader.read(reuse, this.decoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: OperatorStateRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: PostVersionedIOReadableWritable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 
Example #8
Source Project: flink   Author: apache   File: MapSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);

	for (int i = 0; i < size; ++i) {
		keySerializer.copy(source, target);
		
		boolean isNull = source.readBoolean();
		target.writeBoolean(isNull);
		
		if (!isNull) {
			valueSerializer.copy(source, target);
		}
	}
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: PojoSerializerSnapshotData.java    License: Apache License 2.0 5 votes vote down vote up
private static BiFunctionWithException<DataInputView, String, TypeSerializerSnapshot<?>, IOException> snapshotReader(ClassLoader cl) {
	return (input, unused) -> {
		try {
			return TypeSerializerSnapshot.readVersionedSnapshot(input, cl);
		}
		catch (Throwable t) {
			LOG.warn("Exception while reading serializer snapshot.", t);
			return null;
		}
	};
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: TypeSerializerConfigSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public final void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	if (readVersion != ADAPTER_VERSION) {
		throw new IOException("Wrong/unexpected version for the TypeSerializerConfigSnapshot: " + readVersion);
	}

	serializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true);

	// now delegate to the snapshots own reading code
	setUserCodeClassLoader(userCodeClassLoader);
	read(in);
}
 
Example #11
Source Project: flink   Author: apache   File: FlinkKafkaProducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(
	DataInputView source, DataOutputView target) throws IOException {
	boolean hasTransactionalId = source.readBoolean();
	target.writeBoolean(hasTransactionalId);
	if (hasTransactionalId) {
		target.writeUTF(source.readUTF());
	}
	target.writeLong(source.readLong());
	target.writeShort(source.readShort());
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: TupleComparatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	if (deserializedFields1 == null) {
		instantiateDeserializationUtils();
	}
	
	int i = 0;
	try {
		for (; i < serializers.length; i++) {
			deserializedFields1[i] = serializers[i].deserialize(deserializedFields1[i], firstSource);
			deserializedFields2[i] = serializers[i].deserialize(deserializedFields2[i], secondSource);
		}
		
		for (i = 0; i < keyPositions.length; i++) {
			int keyPos = keyPositions[i];
			int cmp = comparators[i].compare(deserializedFields1[keyPos], deserializedFields2[keyPos]);
			if (cmp != 0) {
				return cmp;
			}
		}
		
		return 0;
	} catch (NullPointerException npex) {
		throw new NullKeyFieldException(keyPositions[i]);
	} catch (IndexOutOfBoundsException iobex) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i], iobex);
	}
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: FloatValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readFloat();
	}
}
 
Example #14
Source Project: flink   Author: apache   File: IntPrimitiveArraySerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int[] deserialize(DataInputView source) throws IOException {
	final int len = source.readInt();
	int[] result = new int[len];
	
	for (int i = 0; i < len; i++) {
		result[i] = source.readInt();
	}
	
	return result;
}
 
Example #15
Source Project: flink   Author: apache   File: RowSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int len = fieldSerializers.length;

	// copy bitmask
	readIntoAndCopyMask(source, target, mask);

	// copy non-null fields
	for (int fieldPos = 0; fieldPos < len; fieldPos++) {
		if (!mask[legacyOffset + fieldPos]) {
			fieldSerializers[fieldPos].copy(source, target);
		}
	}
}
 
Example #16
Source Project: flink   Author: apache   File: StringValue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
	int len = in.readUnsignedByte();
	target.writeByte(len);

	if (len >= HIGH_BIT) {
		int shift = 7;
		int curr;
		len = len & 0x7f;
		while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
			len |= (curr & 0x7f) << shift;
			shift += 7;
			target.writeByte(curr);
		}
		len |= curr << shift;
		target.writeByte(curr);
	}

	for (int i = 0; i < len; i++) {
		int c = in.readUnsignedByte();
		target.writeByte(c);
		while (c >= HIGH_BIT) {
			c = in.readUnsignedByte();
			target.writeByte(c);
		}
	}
}
 
Example #17
Source Project: flink   Author: apache   File: CustomIntSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Integer deserialize(DataInputView source) throws IOException {
	int header = source.readInt();
	if (header != MAGIC_VALUE) {
		throw new RuntimeException(String.format("Invalid magic value, expected %d found %d", MAGIC_VALUE, header));
	}
	return source.readInt();
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: LinkedOptionalMapSerializer.java    License: Apache License 2.0 5 votes vote down vote up
public static <K, V> LinkedOptionalMap<K, V> readOptionalMap(
	DataInputView in,
	BiFunctionWithException<DataInputView, String, K, IOException> keyReader,
	BiFunctionWithException<DataInputView, String, V, IOException> valueReader) throws IOException {

	final long header = in.readLong();
	checkState(header == HEADER, "Corrupted stream received header %s", header);

	long mapSize = in.readInt();
	LinkedOptionalMap<K, V> map = new LinkedOptionalMap<>();
	for (int i = 0; i < mapSize; i++) {
		String keyName = in.readUTF();

		final K key;
		if (in.readBoolean()) {
			key = tryReadFrame(in, keyName, keyReader);
		}
		else {
			key = null;
		}

		final V value;
		if (in.readBoolean()) {
			value = tryReadFrame(in, keyName, valueReader);
		}
		else {
			value = null;
		}

		map.put(keyName, key, value);
	}
	return map;
}
 
Example #19
Source Project: flink   Author: apache   File: IntComparator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	int i1 = firstSource.readInt();
	int i2 = secondSource.readInt();
	int comp = (i1 < i2 ? -1 : (i1 == i2 ? 0 : 1)); 
	return ascendingComparison ? comp : -comp; 
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ElementType deserialize(ElementType reuse, DataInputView source) throws IOException {
	if (failOnRead) {
		throw new TestException();
	}
	return new ElementType(source.readInt());
}
 
Example #21
Source Project: flink   Author: flink-tpc-ds   File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readByte();
	}
}
 
Example #22
Source Project: flink   Author: apache   File: ArrayDataSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		DataInputViewStream inStream = new DataInputViewStream(in);
		this.previousType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousEleSer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
}
 
Example #23
Source Project: flink   Author: flink-tpc-ds   File: TypeSerializerSnapshotMigrationTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private void assertSerializerIsAbleToReadOldData(TypeSerializer<ElementT> serializer) throws IOException {
	DataInputView input = dataUnderTest();

	final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
	for (int i = 0; i < testSpecification.testDataCount; i++) {
		final ElementT result = serializer.deserialize(input);
		assertThat(result, matcher);
	}
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: TupleSerializerConfigSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: Kafka09ITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String deserialize(byte[] messageKey, byte[] message, String topic, int partition,
	long offset) throws IOException {
	cnt++;
	DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message));
	String e = ser.deserialize(in);
	return e;
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: NullableSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private boolean deserializeNull(DataInputView source) throws IOException {
	boolean isNull = source.readBoolean();
	if (isNull) {
		source.skipBytesToRead(padding.length);
	}
	return isNull;
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: FlinkKafkaProducer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FlinkKafkaProducer.KafkaTransactionState deserialize(
	FlinkKafkaProducer.KafkaTransactionState reuse,
	DataInputView source) throws IOException {
	return deserialize(source);
}
 
Example #28
Source Project: flink   Author: apache   File: StringValueArray.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	copyInternal(source, target);
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: NullValueComparator.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public NullValue readWithKeyDenormalization(NullValue reuse, DataInputView source) throws IOException {
	throw new UnsupportedOperationException();
}
 
Example #30
Source Project: flink   Author: apache   File: BooleanType.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	this.value = in.readBoolean();
}