org.apache.flink.core.memory.DataOutputView Java Examples

The following examples show how to use org.apache.flink.core.memory.DataOutputView. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: flink-tpc-ds   File: TypeSerializerSerializationUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write a list of serializers and their corresponding config snapshots to the provided
 * data output view. This method writes in a fault tolerant way, so that when read again
 * using {@link #readSerializersAndConfigsWithResilience(DataInputView, ClassLoader)}, if
 * deserialization of the serializer fails, its configuration snapshot will remain intact.
 *
 * <p>Specifically, all written serializers and their config snapshots are indexed by their
 * offset positions within the serialized bytes. The serialization format is as follows:
 * <ul>
 *     <li>1. number of serializer and configuration snapshot pairs.</li>
 *     <li>2. offsets of each serializer and configuration snapshot, in order.</li>
 *     <li>3. total number of bytes for the serialized serializers and the config snapshots.</li>
 *     <li>4. serialized serializers and the config snapshots.</li>
 * </ul>
 *
 * @param out the data output view.
 * @param serializersAndConfigs serializer and configuration snapshot pairs
 *
 * @throws IOException
 */
public static void writeSerializersAndConfigsWithResilience(
		DataOutputView out,
		List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs) throws IOException {

	try (
		ByteArrayOutputStreamWithPos bufferWithPos = new ByteArrayOutputStreamWithPos();
		DataOutputViewStreamWrapper bufferWrapper = new DataOutputViewStreamWrapper(bufferWithPos)) {

		out.writeInt(serializersAndConfigs.size());
		for (Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> serAndConfSnapshot : serializersAndConfigs) {
			out.writeInt(bufferWithPos.getPosition());
			writeSerializer(bufferWrapper, serAndConfSnapshot.f0);

			out.writeInt(bufferWithPos.getPosition());
			TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
				bufferWrapper, (TypeSerializerSnapshot) serAndConfSnapshot.f1, serAndConfSnapshot.f0);
		}

		out.writeInt(bufferWithPos.getPosition());
		out.write(bufferWithPos.getBuf(), 0, bufferWithPos.getPosition());
	}
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serialize(T value, DataOutputView target) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.encoder.setOut(target);
		this.writer.write(value, this.encoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: Record.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int val = source.readUnsignedByte();
	target.writeByte(val);
	
	if (val >= MAX_BIT) {
		int shift = 7;
		int curr;
		val = val & 0x7f;
		while ((curr = source.readUnsignedByte()) >= MAX_BIT) {
			target.writeByte(curr);
			val |= (curr & 0x7f) << shift;
			shift += 7;
		}
		target.writeByte(curr);
		val |= curr << shift;
	}
	
	target.write(source, val);
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: TwoPhaseCommitSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(
		DataInputView source, DataOutputView target) throws IOException {
	TXN pendingTxnHandle = transactionSerializer.deserialize(source);
	transactionSerializer.serialize(pendingTxnHandle, target);
	final long pendingTxnStartTime = source.readLong();
	target.writeLong(pendingTxnStartTime);

	int numPendingCommitTxns = source.readInt();
	target.writeInt(numPendingCommitTxns);
	for (int i = 0; i < numPendingCommitTxns; i++) {
		TXN pendingCommitTxnHandle = transactionSerializer.deserialize(source);
		transactionSerializer.serialize(pendingCommitTxnHandle, target);
		final long pendingCommitTxnStartTime = source.readLong();
		target.writeLong(pendingCommitTxnStartTime);
	}

	boolean hasContext = source.readBoolean();
	target.writeBoolean(hasContext);
	if (hasContext) {
		CONTEXT context = contextSerializer.deserialize(source);
		contextSerializer.serialize(context, target);
	}
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: TwoPhaseCommitSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serialize(
		State<TXN, CONTEXT> record,
		DataOutputView target) throws IOException {
	final TransactionHolder<TXN> pendingTransaction = record.getPendingTransaction();
	transactionSerializer.serialize(pendingTransaction.handle, target);
	target.writeLong(pendingTransaction.transactionStartTime);

	final List<TransactionHolder<TXN>> pendingCommitTransactions = record.getPendingCommitTransactions();
	target.writeInt(pendingCommitTransactions.size());
	for (TransactionHolder<TXN> pendingTxn : pendingCommitTransactions) {
		transactionSerializer.serialize(pendingTxn.handle, target);
		target.writeLong(pendingTxn.transactionStartTime);
	}

	Optional<CONTEXT> context = record.getContext();
	if (context.isPresent()) {
		target.writeBoolean(true);
		contextSerializer.serialize(context.get(), target);
	} else {
		target.writeBoolean(false);
	}
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: RocksDBKeySerializationUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example #7
Source Project: cascading-flink   Author: dataArtisans   File: NullMaskSerDeUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void writeNullMask(
		Object[] array, int length, DataOutputView target) throws IOException {

	int b;
	int bytePos;

	for(int fieldPos = 0; fieldPos < length; ) {
		b = 0x00;
		// set bits in byte
		for(bytePos = 0; bytePos < 8 && fieldPos < length; bytePos++, fieldPos++) {
			b = b << 1;
			// set bit if field is null
			if(array[fieldPos] == null) {
				b |= 0x01;
			}
		}
		// shift bits if last byte is not completely filled
		for(; bytePos < 8; bytePos++) {
			b = b << 1;
		}
		// write byte
		target.writeByte(b);
	}
}
 
Example #8
Source Project: flink   Author: apache   File: KryoRegistrationSerializerConfigSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutputView out) throws IOException {
	super.write(out);

	out.writeInt(kryoRegistrations.size());
	for (Map.Entry<String, KryoRegistration> kryoRegistrationEntry : kryoRegistrations.entrySet()) {
		out.writeUTF(kryoRegistrationEntry.getKey());
		new KryoRegistrationSerializationProxy<>(kryoRegistrationEntry.getValue()).write(out);
	}
}
 
Example #9
Source Project: flink   Author: apache   File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeByte(data[i]);
	}
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: InternalPriorityQueueTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: LinkedOptionalMapSerializer.java    License: Apache License 2.0 5 votes vote down vote up
public static <K, V> void writeOptionalMap(
	DataOutputView out,
	LinkedOptionalMap<K, V> map,
	BiConsumerWithException<DataOutputView, K, IOException> keyWriter,
	BiConsumerWithException<DataOutputView, V, IOException> valueWriter) throws IOException {

	out.writeLong(HEADER);
	out.writeInt(map.size());
	map.forEach(((keyName, key, value) -> {
		out.writeUTF(keyName);

		if (key == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, keyWriter, key);
		}

		if (value == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, valueWriter, value);
		}
	}));
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
protected static void copyInternal(DataInputView source, DataOutputView target) throws IOException {
	int count = source.readInt();
	target.writeInt(count);

	int bytes = ELEMENT_LENGTH_IN_BYTES * count;
	target.write(source, bytes);
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: StatefulComplexPayloadSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(ComplexPayload record, DataOutputView target) throws IOException {
	try {
		if (currentOwnerThread.compareAndSet(null, Thread.currentThread())) {
			target.write(InstantiationUtil.serializeObject(record));
		} else {
			throw new IllegalStateException("Concurrent access to type serializer detected!");
		}
	} finally {
		currentOwnerThread.set(null);
	}
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: DeweyNumber.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);
	for (int i = 0; i < size; i++) {
		target.writeInt(source.readInt());
	}
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: BigDecSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final boolean isNull = BigIntSerializer.copyBigInteger(source, target);
	if (!isNull) {
		final int scale = source.readInt();
		target.writeInt(scale);
	}
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: KafkaConsumerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public byte[] serializeValue(Tuple3<Integer, Integer, String> element) {
	ByteArrayOutputStream by = new ByteArrayOutputStream();
	DataOutputView out = new DataOutputViewStreamWrapper(by);
	try {
		ts.serialize(new Tuple2<>(element.f0, element.f1), out);
	} catch (IOException e) {
		throw new RuntimeException("Error" , e);
	}
	return by.toByteArray();
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: BinaryRowSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(BinaryRow record, DataOutputView target) throws IOException {
	target.writeInt(record.getSizeInBytes());
	if (target instanceof MemorySegmentWritable) {
		serializeWithoutLength(record, (MemorySegmentWritable) target);
	} else {
		SegmentsUtil.copyToView(
				record.getSegments(), record.getOffset(),
				record.getSizeInBytes(), target);
	}
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: DecimalSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	if (Decimal.isCompact(precision)) {
		target.writeLong(source.readLong());
	} else {
		int len = source.readInt();
		target.writeInt(len);
		target.write(source, len);
	}
}
 
Example #19
Source Project: flink-benchmarks   Author: dataArtisans   File: StringSerializationBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@Benchmark
public byte[] stringWrite() throws IOException {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputView out = new DataOutputViewStreamWrapper(buffer);
    serializer.serialize(input, out);
    return buffer.toByteArray();
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void copyStates(DataInputView source, DataOutputView target) throws IOException {
	int computationStateNo = source.readInt();
	target.writeInt(computationStateNo);

	for (int i = 0; i < computationStateNo; i++) {
		copySingleComputationState(source, target);
	}
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: IntListSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	target.writeInt(source.readInt());
	int len = source.readInt();
	target.writeInt(len);
	for (int i = 0; i < len; i++) {
		target.writeInt(source.readInt());
	}
}
 
Example #22
Source Project: bahir-flink   Author: apache   File: SiddhiStreamOperator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void snapshotQueueState(PriorityQueue<StreamRecord<Tuple2<String, IN>>> queue, DataOutputView dataOutputView) throws IOException {
    dataOutputView.writeInt(queue.size());
    for (StreamRecord<Tuple2<String, IN>> record : queue) {
        String streamId = record.getValue().f0;
        dataOutputView.writeUTF(streamId);
        this.getStreamRecordSerializer(streamId).serialize(record, dataOutputView);
    }
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: IntValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeInt(data[i]);
	}
}
 
Example #24
Source Project: flink   Author: apache   File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example #25
Source Project: flink   Author: apache   File: InternalTimersSnapshotReaderWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public final void writeTimersSnapshot(DataOutputView out) throws IOException {
	writeKeyAndNamespaceSerializers(out);

	LegacyTimerSerializer<K, N> timerSerializer = new LegacyTimerSerializer<>(
		keySerializer,
		namespaceSerializer);

	// write the event time timers
	Set<TimerHeapInternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
	if (eventTimers != null) {
		out.writeInt(eventTimers.size());
		for (TimerHeapInternalTimer<K, N> eventTimer : eventTimers) {
			timerSerializer.serialize(eventTimer, out);
		}
	} else {
		out.writeInt(0);
	}

	// write the processing time timers
	Set<TimerHeapInternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
	if (processingTimers != null) {
		out.writeInt(processingTimers.size());
		for (TimerHeapInternalTimer<K, N> processingTimer : processingTimers) {
			timerSerializer.serialize(processingTimer, out);
		}
	} else {
		out.writeInt(0);
	}
}
 
Example #26
Source Project: flink   Author: apache   File: InternalPriorityQueueTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: TestData.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void writeWithKeyNormalization(IntPair record, DataOutputView target) throws IOException {
	target.writeInt(record.getKey() - Integer.MIN_VALUE);
	target.writeInt(record.getValue());
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: UnloadableDummyTypeSerializer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void serialize(T record, DataOutputView target) throws IOException {
	throw new UnsupportedOperationException("This object is a dummy TypeSerializer.");
}
 
Example #29
Source Project: flink   Author: apache   File: CollectionInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	target.writeInt(source.readInt());
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: CompositeTypeSerializerSnapshot.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final void writeSnapshot(DataOutputView out) throws IOException {
	internalWriteOuterSnapshot(out);
	nestedSerializersSnapshotDelegate.writeNestedSerializerSnapshots(out);
}