Java Code Examples for org.apache.flink.core.memory.DataOutputView

The following examples show how to use org.apache.flink.core.memory.DataOutputView. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: TypeSerializerSerializationUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write a list of serializers and their corresponding config snapshots to the provided
 * data output view. This method writes in a fault tolerant way, so that when read again
 * using {@link #readSerializersAndConfigsWithResilience(DataInputView, ClassLoader)}, if
 * deserialization of the serializer fails, its configuration snapshot will remain intact.
 *
 * <p>Specifically, all written serializers and their config snapshots are indexed by their
 * offset positions within the serialized bytes. The serialization format is as follows:
 * <ul>
 *     <li>1. number of serializer and configuration snapshot pairs.</li>
 *     <li>2. offsets of each serializer and configuration snapshot, in order.</li>
 *     <li>3. total number of bytes for the serialized serializers and the config snapshots.</li>
 *     <li>4. serialized serializers and the config snapshots.</li>
 * </ul>
 *
 * @param out the data output view.
 * @param serializersAndConfigs serializer and configuration snapshot pairs
 *
 * @throws IOException
 */
public static void writeSerializersAndConfigsWithResilience(
		DataOutputView out,
		List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs) throws IOException {

	try (
		ByteArrayOutputStreamWithPos bufferWithPos = new ByteArrayOutputStreamWithPos();
		DataOutputViewStreamWrapper bufferWrapper = new DataOutputViewStreamWrapper(bufferWithPos)) {

		out.writeInt(serializersAndConfigs.size());
		for (Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> serAndConfSnapshot : serializersAndConfigs) {
			out.writeInt(bufferWithPos.getPosition());
			writeSerializer(bufferWrapper, serAndConfSnapshot.f0);

			out.writeInt(bufferWithPos.getPosition());
			TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
				bufferWrapper, (TypeSerializerSnapshot) serAndConfSnapshot.f1, serAndConfSnapshot.f0);
		}

		out.writeInt(bufferWithPos.getPosition());
		out.write(bufferWithPos.getBuf(), 0, bufferWithPos.getPosition());
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serialize(T value, DataOutputView target) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.encoder.setOut(target);
		this.writer.write(value, this.encoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 
Example 3
Source Project: flink   Source File: Record.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int val = source.readUnsignedByte();
	target.writeByte(val);
	
	if (val >= MAX_BIT) {
		int shift = 7;
		int curr;
		val = val & 0x7f;
		while ((curr = source.readUnsignedByte()) >= MAX_BIT) {
			target.writeByte(curr);
			val |= (curr & 0x7f) << shift;
			shift += 7;
		}
		target.writeByte(curr);
		val |= curr << shift;
	}
	
	target.write(source, val);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: TwoPhaseCommitSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copy(
		DataInputView source, DataOutputView target) throws IOException {
	TXN pendingTxnHandle = transactionSerializer.deserialize(source);
	transactionSerializer.serialize(pendingTxnHandle, target);
	final long pendingTxnStartTime = source.readLong();
	target.writeLong(pendingTxnStartTime);

	int numPendingCommitTxns = source.readInt();
	target.writeInt(numPendingCommitTxns);
	for (int i = 0; i < numPendingCommitTxns; i++) {
		TXN pendingCommitTxnHandle = transactionSerializer.deserialize(source);
		transactionSerializer.serialize(pendingCommitTxnHandle, target);
		final long pendingCommitTxnStartTime = source.readLong();
		target.writeLong(pendingCommitTxnStartTime);
	}

	boolean hasContext = source.readBoolean();
	target.writeBoolean(hasContext);
	if (hasContext) {
		CONTEXT context = contextSerializer.deserialize(source);
		contextSerializer.serialize(context, target);
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: TwoPhaseCommitSinkFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serialize(
		State<TXN, CONTEXT> record,
		DataOutputView target) throws IOException {
	final TransactionHolder<TXN> pendingTransaction = record.getPendingTransaction();
	transactionSerializer.serialize(pendingTransaction.handle, target);
	target.writeLong(pendingTransaction.transactionStartTime);

	final List<TransactionHolder<TXN>> pendingCommitTransactions = record.getPendingCommitTransactions();
	target.writeInt(pendingCommitTransactions.size());
	for (TransactionHolder<TXN> pendingTxn : pendingCommitTransactions) {
		transactionSerializer.serialize(pendingTxn.handle, target);
		target.writeLong(pendingTxn.transactionStartTime);
	}

	Optional<CONTEXT> context = record.getContext();
	if (context.isPresent()) {
		target.writeBoolean(true);
		contextSerializer.serialize(context.get(), target);
	} else {
		target.writeBoolean(false);
	}
}
 
Example 6
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example 7
Source Project: cascading-flink   Source File: NullMaskSerDeUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void writeNullMask(
		Object[] array, int length, DataOutputView target) throws IOException {

	int b;
	int bytePos;

	for(int fieldPos = 0; fieldPos < length; ) {
		b = 0x00;
		// set bits in byte
		for(bytePos = 0; bytePos < 8 && fieldPos < length; bytePos++, fieldPos++) {
			b = b << 1;
			// set bit if field is null
			if(array[fieldPos] == null) {
				b |= 0x01;
			}
		}
		// shift bits if last byte is not completely filled
		for(; bytePos < 8; bytePos++) {
			b = b << 1;
		}
		// write byte
		target.writeByte(b);
	}
}
 
Example 8
@Override
public void write(DataOutputView out) throws IOException {
	super.write(out);

	out.writeInt(kryoRegistrations.size());
	for (Map.Entry<String, KryoRegistration> kryoRegistrationEntry : kryoRegistrations.entrySet()) {
		out.writeUTF(kryoRegistrationEntry.getKey());
		new KryoRegistrationSerializationProxy<>(kryoRegistrationEntry.getValue()).write(out);
	}
}
 
Example 9
Source Project: flink   Source File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeByte(data[i]);
	}
}
 
Example 10
Source Project: flink   Source File: InternalPriorityQueueTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 
Example 11
Source Project: flink   Source File: LinkedOptionalMapSerializer.java    License: Apache License 2.0 5 votes vote down vote up
public static <K, V> void writeOptionalMap(
	DataOutputView out,
	LinkedOptionalMap<K, V> map,
	BiConsumerWithException<DataOutputView, K, IOException> keyWriter,
	BiConsumerWithException<DataOutputView, V, IOException> valueWriter) throws IOException {

	out.writeLong(HEADER);
	out.writeInt(map.size());
	map.forEach(((keyName, key, value) -> {
		out.writeUTF(keyName);

		if (key == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, keyWriter, key);
		}

		if (value == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, valueWriter, value);
		}
	}));
}
 
Example 12
Source Project: Flink-CEPplus   Source File: ByteValueArray.java    License: Apache License 2.0 5 votes vote down vote up
protected static void copyInternal(DataInputView source, DataOutputView target) throws IOException {
	int count = source.readInt();
	target.writeInt(count);

	int bytes = ELEMENT_LENGTH_IN_BYTES * count;
	target.write(source, bytes);
}
 
Example 13
Source Project: flink   Source File: StatefulComplexPayloadSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(ComplexPayload record, DataOutputView target) throws IOException {
	try {
		if (currentOwnerThread.compareAndSet(null, Thread.currentThread())) {
			target.write(InstantiationUtil.serializeObject(record));
		} else {
			throw new IllegalStateException("Concurrent access to type serializer detected!");
		}
	} finally {
		currentOwnerThread.set(null);
	}
}
 
Example 14
Source Project: flink   Source File: DeweyNumber.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);
	for (int i = 0; i < size; i++) {
		target.writeInt(source.readInt());
	}
}
 
Example 15
Source Project: flink   Source File: BigDecSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final boolean isNull = BigIntSerializer.copyBigInteger(source, target);
	if (!isNull) {
		final int scale = source.readInt();
		target.writeInt(scale);
	}
}
 
Example 16
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public byte[] serializeValue(Tuple3<Integer, Integer, String> element) {
	ByteArrayOutputStream by = new ByteArrayOutputStream();
	DataOutputView out = new DataOutputViewStreamWrapper(by);
	try {
		ts.serialize(new Tuple2<>(element.f0, element.f1), out);
	} catch (IOException e) {
		throw new RuntimeException("Error" , e);
	}
	return by.toByteArray();
}
 
Example 17
Source Project: flink   Source File: BinaryRowSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(BinaryRow record, DataOutputView target) throws IOException {
	target.writeInt(record.getSizeInBytes());
	if (target instanceof MemorySegmentWritable) {
		serializeWithoutLength(record, (MemorySegmentWritable) target);
	} else {
		SegmentsUtil.copyToView(
				record.getSegments(), record.getOffset(),
				record.getSizeInBytes(), target);
	}
}
 
Example 18
Source Project: flink   Source File: DecimalSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	if (Decimal.isCompact(precision)) {
		target.writeLong(source.readLong());
	} else {
		int len = source.readInt();
		target.writeInt(len);
		target.write(source, len);
	}
}
 
Example 19
@Benchmark
public byte[] stringWrite() throws IOException {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputView out = new DataOutputViewStreamWrapper(buffer);
    serializer.serialize(input, out);
    return buffer.toByteArray();
}
 
Example 20
Source Project: Flink-CEPplus   Source File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void copyStates(DataInputView source, DataOutputView target) throws IOException {
	int computationStateNo = source.readInt();
	target.writeInt(computationStateNo);

	for (int i = 0; i < computationStateNo; i++) {
		copySingleComputationState(source, target);
	}
}
 
Example 21
Source Project: Flink-CEPplus   Source File: IntListSerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	target.writeInt(source.readInt());
	int len = source.readInt();
	target.writeInt(len);
	for (int i = 0; i < len; i++) {
		target.writeInt(source.readInt());
	}
}
 
Example 22
Source Project: bahir-flink   Source File: SiddhiStreamOperator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void snapshotQueueState(PriorityQueue<StreamRecord<Tuple2<String, IN>>> queue, DataOutputView dataOutputView) throws IOException {
    dataOutputView.writeInt(queue.size());
    for (StreamRecord<Tuple2<String, IN>> record : queue) {
        String streamId = record.getValue().f0;
        dataOutputView.writeUTF(streamId);
        this.getStreamRecordSerializer(streamId).serialize(record, dataOutputView);
    }
}
 
Example 23
Source Project: Flink-CEPplus   Source File: IntValueArray.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeInt(data[i]);
	}
}
 
Example 24
Source Project: flink   Source File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example 25
Source Project: flink   Source File: InternalTimersSnapshotReaderWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public final void writeTimersSnapshot(DataOutputView out) throws IOException {
	writeKeyAndNamespaceSerializers(out);

	LegacyTimerSerializer<K, N> timerSerializer = new LegacyTimerSerializer<>(
		keySerializer,
		namespaceSerializer);

	// write the event time timers
	Set<TimerHeapInternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
	if (eventTimers != null) {
		out.writeInt(eventTimers.size());
		for (TimerHeapInternalTimer<K, N> eventTimer : eventTimers) {
			timerSerializer.serialize(eventTimer, out);
		}
	} else {
		out.writeInt(0);
	}

	// write the processing time timers
	Set<TimerHeapInternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
	if (processingTimers != null) {
		out.writeInt(processingTimers.size());
		for (TimerHeapInternalTimer<K, N> processingTimer : processingTimers) {
			timerSerializer.serialize(processingTimer, out);
		}
	} else {
		out.writeInt(0);
	}
}
 
Example 26
Source Project: flink   Source File: InternalPriorityQueueTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 
Example 27
Source Project: flink   Source File: TestData.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void writeWithKeyNormalization(IntPair record, DataOutputView target) throws IOException {
	target.writeInt(record.getKey() - Integer.MIN_VALUE);
	target.writeInt(record.getValue());
}
 
Example 28
Source Project: flink   Source File: UnloadableDummyTypeSerializer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void serialize(T record, DataOutputView target) throws IOException {
	throw new UnsupportedOperationException("This object is a dummy TypeSerializer.");
}
 
Example 29
Source Project: flink   Source File: CollectionInputFormatTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	target.writeInt(source.readInt());
}
 
Example 30
Source Project: flink   Source File: CompositeTypeSerializerSnapshot.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final void writeSnapshot(DataOutputView out) throws IOException {
	internalWriteOuterSnapshot(out);
	nestedSerializersSnapshotDelegate.writeNestedSerializerSnapshots(out);
}