org.apache.flink.core.memory.DataInputView Java Examples

The following examples show how to use org.apache.flink.core.memory.DataInputView. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PostVersionedIOReadableWritable.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 
Example #2
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
 
Example #3
Source File: InternalTimersSnapshotReaderWriters.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
protected void restoreKeyAndNamespaceSerializers(
		InternalTimersSnapshot<K, N> restoredTimersSnapshot,
		DataInputView in) throws IOException {

	DataInputViewStream dis = new DataInputViewStream(in);
	try {
		final TypeSerializer<K> keySerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);
		final TypeSerializer<N> namespaceSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);

		restoredTimersSnapshot.setKeySerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(keySerializer));
		restoredTimersSnapshot.setNamespaceSerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(namespaceSerializer));
	} catch (ClassNotFoundException exception) {
		throw new IOException(exception);
	}
}
 
Example #4
Source File: MapSerializer.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);

	for (int i = 0; i < size; ++i) {
		keySerializer.copy(source, target);
		
		boolean isNull = source.readBoolean();
		target.writeBoolean(isNull);
		
		if (!isNull) {
			valueSerializer.copy(source, target);
		}
	}
}
 
Example #5
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 
Example #6
Source File: InternalTimerServiceSerializationProxy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected void read(DataInputView in, boolean wasVersioned) throws IOException {
	int noOfTimerServices = in.readInt();

	for (int i = 0; i < noOfTimerServices; i++) {
		String serviceName = in.readUTF();

		int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
		InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
			.getReaderForVersion(readerVersion, userCodeClassLoader)
			.readTimersSnapshot(in);

		InternalTimerServiceImpl<K, ?> timerService = registerOrGetTimerService(
			serviceName,
			restoredTimersSnapshot);

		timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
	}
}
 
Example #7
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example #8
Source File: AvroSerializer.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.decoder.setIn(source);
		return this.reader.read(reuse, this.decoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 
Example #9
Source File: PojoSerializerSnapshotData.java    From flink with Apache License 2.0 5 votes vote down vote up
private static BiFunctionWithException<DataInputView, String, TypeSerializerSnapshot<?>, IOException> snapshotReader(ClassLoader cl) {
	return (input, unused) -> {
		try {
			return TypeSerializerSnapshot.readVersionedSnapshot(input, cl);
		}
		catch (Throwable t) {
			LOG.warn("Exception while reading serializer snapshot.", t);
			return null;
		}
	};
}
 
Example #10
Source File: FloatValueArray.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readFloat();
	}
}
 
Example #11
Source File: TypeSerializerConfigSnapshot.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public final void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	if (readVersion != ADAPTER_VERSION) {
		throw new IOException("Wrong/unexpected version for the TypeSerializerConfigSnapshot: " + readVersion);
	}

	serializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true);

	// now delegate to the snapshots own reading code
	setUserCodeClassLoader(userCodeClassLoader);
	read(in);
}
 
Example #12
Source File: TupleSerializerConfigSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}
 
Example #13
Source File: CustomIntSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Integer deserialize(DataInputView source) throws IOException {
	int header = source.readInt();
	if (header != MAGIC_VALUE) {
		throw new RuntimeException(String.format("Invalid magic value, expected %d found %d", MAGIC_VALUE, header));
	}
	return source.readInt();
}
 
Example #14
Source File: TupleComparatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	if (deserializedFields1 == null) {
		instantiateDeserializationUtils();
	}
	
	int i = 0;
	try {
		for (; i < serializers.length; i++) {
			deserializedFields1[i] = serializers[i].deserialize(deserializedFields1[i], firstSource);
			deserializedFields2[i] = serializers[i].deserialize(deserializedFields2[i], secondSource);
		}
		
		for (i = 0; i < keyPositions.length; i++) {
			int keyPos = keyPositions[i];
			int cmp = comparators[i].compare(deserializedFields1[keyPos], deserializedFields2[keyPos]);
			if (cmp != 0) {
				return cmp;
			}
		}
		
		return 0;
	} catch (NullPointerException npex) {
		throw new NullKeyFieldException(keyPositions[i]);
	} catch (IndexOutOfBoundsException iobex) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i], iobex);
	}
}
 
Example #15
Source File: Kafka09ITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public String deserialize(byte[] messageKey, byte[] message, String topic, int partition,
	long offset) throws IOException {
	cnt++;
	DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message));
	String e = ser.deserialize(in);
	return e;
}
 
Example #16
Source File: NullableSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean deserializeNull(DataInputView source) throws IOException {
	boolean isNull = source.readBoolean();
	if (isNull) {
		source.skipBytesToRead(padding.length);
	}
	return isNull;
}
 
Example #17
Source File: FlinkKafkaProducer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void copy(
	DataInputView source, DataOutputView target) throws IOException {
	boolean hasTransactionalId = source.readBoolean();
	target.writeBoolean(hasTransactionalId);
	if (hasTransactionalId) {
		target.writeUTF(source.readUTF());
	}
	target.writeLong(source.readLong());
	target.writeShort(source.readShort());
}
 
Example #18
Source File: IntPrimitiveArraySerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public int[] deserialize(DataInputView source) throws IOException {
	final int len = source.readInt();
	int[] result = new int[len];
	
	for (int i = 0; i < len; i++) {
		result[i] = source.readInt();
	}
	
	return result;
}
 
Example #19
Source File: TypeSerializerSnapshotMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void assertSerializerIsAbleToReadOldData(TypeSerializer<ElementT> serializer) throws IOException {
	DataInputView input = dataUnderTest();

	final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
	for (int i = 0; i < testSpecification.testDataCount; i++) {
		final ElementT result = serializer.deserialize(input);
		assertThat(result, matcher);
	}
}
 
Example #20
Source File: ArrayDataSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		DataInputViewStream inStream = new DataInputViewStream(in);
		this.previousType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousEleSer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
}
 
Example #21
Source File: StringValue.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
	int len = in.readUnsignedByte();
	target.writeByte(len);

	if (len >= HIGH_BIT) {
		int shift = 7;
		int curr;
		len = len & 0x7f;
		while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
			len |= (curr & 0x7f) << shift;
			shift += 7;
			target.writeByte(curr);
		}
		len |= curr << shift;
		target.writeByte(curr);
	}

	for (int i = 0; i < len; i++) {
		int c = in.readUnsignedByte();
		target.writeByte(c);
		while (c >= HIGH_BIT) {
			c = in.readUnsignedByte();
			target.writeByte(c);
		}
	}
}
 
Example #22
Source File: ByteValueArray.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readByte();
	}
}
 
Example #23
Source File: CollectionInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public ElementType deserialize(ElementType reuse, DataInputView source) throws IOException {
	if (failOnRead) {
		throw new TestException();
	}
	return new ElementType(source.readInt());
}
 
Example #24
Source File: IntComparator.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	int i1 = firstSource.readInt();
	int i2 = secondSource.readInt();
	int comp = (i1 < i2 ? -1 : (i1 == i2 ? 0 : 1)); 
	return ascendingComparison ? comp : -comp; 
}
 
Example #25
Source File: LinkedOptionalMapSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <K, V> LinkedOptionalMap<K, V> readOptionalMap(
	DataInputView in,
	BiFunctionWithException<DataInputView, String, K, IOException> keyReader,
	BiFunctionWithException<DataInputView, String, V, IOException> valueReader) throws IOException {

	final long header = in.readLong();
	checkState(header == HEADER, "Corrupted stream received header %s", header);

	long mapSize = in.readInt();
	LinkedOptionalMap<K, V> map = new LinkedOptionalMap<>();
	for (int i = 0; i < mapSize; i++) {
		String keyName = in.readUTF();

		final K key;
		if (in.readBoolean()) {
			key = tryReadFrame(in, keyName, keyReader);
		}
		else {
			key = null;
		}

		final V value;
		if (in.readBoolean()) {
			value = tryReadFrame(in, keyName, valueReader);
		}
		else {
			value = null;
		}

		map.put(keyName, key, value);
	}
	return map;
}
 
Example #26
Source File: RowSerializer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int len = fieldSerializers.length;

	// copy bitmask
	readIntoAndCopyMask(source, target, mask);

	// copy non-null fields
	for (int fieldPos = 0; fieldPos < len; fieldPos++) {
		if (!mask[legacyOffset + fieldPos]) {
			fieldSerializers[fieldPos].copy(source, target);
		}
	}
}
 
Example #27
Source File: TupleSerializerBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	for (int i = 0; i < arity; i++) {
		fieldSerializers[i].copy(source, target);
	}
}
 
Example #28
Source File: WritableSerializer.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	reuse.readFields(source);
	return reuse;
}
 
Example #29
Source File: ByteValueArraySerializer.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public ByteValueArray deserialize(ByteValueArray reuse, DataInputView source) throws IOException {
	reuse.read(source);
	return reuse;
}
 
Example #30
Source File: CustomSerializationITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void read(DataInputView in) throws IOException {
	// read 8 bytes
	in.readLong();
}