org.apache.flink.core.memory.DataInputViewStreamWrapper Java Examples

The following examples show how to use org.apache.flink.core.memory.DataInputViewStreamWrapper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: VersionedIOWriteableTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadCompatibleVersion() throws Exception {

	String payload = "test";

	TestWriteable testWriteable = new TestWriteable(1, payload);
	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		testWriteable.write(new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	testWriteable = new TestWriteable(2) {
		@Override
		public int[] getCompatibleVersions() {
			return new int[] {1, 2};
		}
	};
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		testWriteable.read(new DataInputViewStreamWrapper(in));
	}

	Assert.assertEquals(payload, testWriteable.getData());
}
 
Example #2
Source File: StateInitializationContextImplTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void getOperatorStateStore() throws Exception {

	Set<Integer> readStatesCount = new HashSet<>();

	for (StatePartitionStreamProvider statePartitionStreamProvider
			: initializationContext.getRawOperatorStateInputs()) {

		Assert.assertNotNull(statePartitionStreamProvider);

		try (InputStream is = statePartitionStreamProvider.getStream()) {
			DataInputView div = new DataInputViewStreamWrapper(is);
			Assert.assertTrue(readStatesCount.add(div.readInt()));
		}
	}

	Assert.assertEquals(writtenOperatorStates, readStatesCount);
}
 
Example #3
Source File: OperatorStateOutputCheckpointStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions) throws IOException {
	int count = 0;
	try (FSDataInputStream in = fullHandle.openInputStream()) {
		OperatorStateHandle.StateMetaInfo metaInfo = fullHandle.getStateNameToPartitionOffsets().
				get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		long[] offsets = metaInfo.getOffsets();

		Assert.assertNotNull(offsets);

		DataInputView div = new DataInputViewStreamWrapper(in);
		for (int i = 0; i < numPartitions; ++i) {
			in.seek(offsets[i]);
			Assert.assertEquals(i, div.readInt());
			++count;
		}
	}

	Assert.assertEquals(numPartitions, count);
}
 
Example #4
Source File: FoldApplyProcessAllWindowFunction.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyAllWindowContext<>(windowFunction);

}
 
Example #5
Source File: RocksDBKeySerializationUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 
Example #6
Source File: StateInitializationContextImplTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void getKeyedStateStreams() throws Exception {

	int readKeyGroupCount = 0;

	for (KeyGroupStatePartitionStreamProvider stateStreamProvider
			: initializationContext.getRawKeyedStateInputs()) {

		Assert.assertNotNull(stateStreamProvider);

		try (InputStream is = stateStreamProvider.getStream()) {
			DataInputView div = new DataInputViewStreamWrapper(is);
			int val = div.readInt();
			++readKeyGroupCount;
			Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val);
		}
	}

	Assert.assertEquals(writtenKeyGroups, readKeyGroupCount);
}
 
Example #7
Source File: StringValueSerializationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void testSerialization(String[] values) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
	DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos);
	
	for (String value : values) {
		StringValue sv = new StringValue(value);
		sv.write(serializer);
	}
	
	serializer.close();
	baos.close();
	
	ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	DataInputViewStreamWrapper deserializer = new DataInputViewStreamWrapper(bais);
	
	int num = 0;
	while (bais.available() > 0) {
		StringValue deser = new StringValue();
		deser.read(deserializer);
		
		assertEquals("DeserializedString differs from original string.", values[num], deser.getValue());
		num++;
	}
	
	assertEquals("Wrong number of deserialized values", values.length, num);
}
 
Example #8
Source File: CollectionInputFormat.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
	in.defaultReadObject();

	int collectionLength = in.readInt();
	List<T> list = new ArrayList<T>(collectionLength);

	if (collectionLength > 0) {
		try {
			DataInputViewStreamWrapper wrapper = new DataInputViewStreamWrapper(in);
			for (int i = 0; i < collectionLength; i++){
				T element = serializer.deserialize(wrapper);
				list.add(element);
			}
		}
		catch (Throwable t) {
			throw new IOException("Error while deserializing element from collection", t);
		}
	}

	dataSet = list;
}
 
Example #9
Source File: FoldApplyAllWindowFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration configuration) throws Exception {
	super.open(configuration);

	if (accSerializer == null) {
		throw new RuntimeException("No serializer set for the fold accumulator type. " +
			"Probably the setOutputType method was not called.");
	}

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);
}
 
Example #10
Source File: StringSerializationBenchmark.java    From flink-benchmarks with Apache License 2.0 6 votes vote down vote up
@Setup
public void setup() throws IOException {
    length = Integer.parseInt(lengthStr);
    switch (type) {
        case "ascii":
            input = generate(asciiChars, length);
            break;
        case "russian":
            input = generate(russianChars, length);
            break;
        case "chinese":
            input = generate(chineseChars, length);
            break;
        default:
            throw new IllegalArgumentException(type + "charset is not supported");
    }
    byte[] stringBytes = stringWrite();
    serializedBuffer = new ByteArrayInputStream(stringBytes);
    serializedStream = new DataInputViewStreamWrapper(serializedBuffer);
}
 
Example #11
Source File: EventWithAggregatorsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example #12
Source File: KryoSerializerCompatibilityTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testMigrationStrategyForRemovedAvroDependency() throws Exception {
	KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig());

	// read configuration again from bytes
	TypeSerializerSnapshot kryoSerializerConfigSnapshot;
	try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) {
		kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA);
	}

	@SuppressWarnings("unchecked")
	TypeSerializerSchemaCompatibility<TestClass> compatResult =
		kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA);
	assertTrue(compatResult.isCompatibleAsIs());
}
 
Example #13
Source File: EventWithAggregatorsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) {
	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		event.write(new DataOutputViewStreamWrapper(baos));

		byte[] data = baos.toByteArray();
		baos.close();

		DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data));
		IterationEventWithAggregators newEvent = event.getClass().newInstance();
		newEvent.read(in);
		in.close();

		return newEvent;
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail("Test threw an exception: " + e.getMessage());
		return null;
	}
}
 
Example #14
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that reading and writing serializers work correctly.
 */
@Test
public void testSerializerSerialization() throws Exception {

	TypeSerializer<?> serializer = IntSerializer.INSTANCE;

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer);
		serialized = out.toByteArray();
	}

	TypeSerializer<?> deserializedSerializer;
	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(serializer, deserializedSerializer);
}
 
Example #15
Source File: ValueStateValueReader.java    From bravo with Apache License 2.0 6 votes vote down vote up
@Override
public void flatMap(KeyedStateRow row, Collector<V> out) throws Exception {
	if (!stateName.equals(row.getStateName())) {
		return;
	}

	byte[] valueBytes = row.getValueBytes();

	V value = null;
	try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) {
		DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs);
		skipTimestampIfTtlEnabled(iw);
		value = valueDeserializer.deserialize(iw);
	}
	if (value == null) {
		throw new RuntimeException("MapStates with null values are not supported at the moment.");
	} else {
		out.collect(value);
	}
}
 
Example #16
Source File: StringValueSerializationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void testSerialization(String[] values) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
	DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos);
	
	for (String value : values) {
		StringValue sv = new StringValue(value);
		sv.write(serializer);
	}
	
	serializer.close();
	baos.close();
	
	ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	DataInputViewStreamWrapper deserializer = new DataInputViewStreamWrapper(bais);
	
	int num = 0;
	while (bais.available() > 0) {
		StringValue deser = new StringValue();
		deser.read(deserializer);
		
		assertEquals("DeserializedString differs from original string.", values[num], deser.getValue());
		num++;
	}
	
	assertEquals("Wrong number of deserialized values", values.length, num);
}
 
Example #17
Source File: PostVersionedIOReadableWritable.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 
Example #18
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that reading and writing configuration snapshots work correctly.
 */
@Test
public void testSerializeConfigurationSnapshots() throws Exception {
	TypeSerializerSerializationUtilTest.TestConfigSnapshot<String> configSnapshot1 =
		new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(1, "foo");

	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			configSnapshot1,
			StringSerializer.INSTANCE);

		serializedConfig = out.toByteArray();
	}

	TypeSerializerSnapshot<?> restoredConfigs;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		restoredConfigs = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), null);
	}

	assertEquals(configSnapshot1, restoredConfigs);
}
 
Example #19
Source File: TypeSerializerSerializationUtilTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that deserializing config snapshots fail if the config class could not be found.
 */
@Test(expected = IOException.class)
public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception {
	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"),
			StringSerializer.INSTANCE);
		serializedConfig = out.toByteArray();
	}

	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		// read using a dummy classloader
		TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null);
	}

	fail("Expected a ClassNotFoundException wrapped in IOException");
}
 
Example #20
Source File: PostVersionedIOReadableWritable.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 
Example #21
Source File: TypeSerializerSerializationUtilTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that reading and writing configuration snapshots work correctly.
 */
@Test
public void testSerializeConfigurationSnapshots() throws Exception {
	TypeSerializerSerializationUtilTest.TestConfigSnapshot<String> configSnapshot1 =
		new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(1, "foo");

	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out),
			configSnapshot1,
			StringSerializer.INSTANCE);

		serializedConfig = out.toByteArray();
	}

	TypeSerializerSnapshot<?> restoredConfigs;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		restoredConfigs = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), null);
	}

	assertEquals(configSnapshot1, restoredConfigs);
}
 
Example #22
Source File: InstantiationUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Clones the given writable using the {@link IOReadableWritable serialization}.
 *
 * @param original Object to clone
 * @param <T> Type of the object to clone
 * @return Cloned object
 * @throws IOException Thrown is the serialization fails.
 */
public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException {
	if (original == null) {
		return null;
	}

	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		original.write(out);
	}

	final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
	try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {

		@SuppressWarnings("unchecked")
		T copy = (T) instantiate(original.getClass());
		copy.read(in);
		return copy;
	}
}
 
Example #23
Source File: OperatorStateRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 
Example #24
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
	// read the non-transient fields
	in.defaultReadObject();

	// read the default value field
	boolean hasDefaultValue = in.readBoolean();
	if (hasDefaultValue) {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		int size = in.readInt();

		byte[] buffer = new byte[size];

		in.readFully(buffer);

		try (ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
				DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bais)) {

			defaultValue = serializer.deserialize(inView);
		}
		catch (Exception e) {
			throw new IOException("Unable to deserialize default value.", e);
		}
	} else {
		defaultValue = null;
	}
}
 
Example #25
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
 
Example #26
Source File: SerializerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSnapshotConfigurationAndReconfigure() throws Exception {
	final TypeSerializer<T> serializer = getSerializer();
	final TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration();

	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), configSnapshot, serializer);
		serializedConfig = out.toByteArray();
	}

	TypeSerializerSnapshot<T> restoredConfig;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), getSerializer());
	}

	TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer());
	final TypeSerializer<T> restoreSerializer;
	if (strategy.isCompatibleAsIs()) {
		restoreSerializer = restoredConfig.restoreSerializer();
	}
	else if (strategy.isCompatibleWithReconfiguredSerializer()) {
		restoreSerializer = strategy.getReconfiguredSerializer();
	}
	else {
		throw new AssertionError("Unable to restore serializer with " + strategy);
	}
	assertEquals(serializer.getClass(), restoreSerializer.getClass());
}
 
Example #27
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Restore one key groups state handle.
 */
private void restoreKeyGroupsInStateHandle()
	throws IOException, StateMigrationException, RocksDBException {
	try {
		currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(currentStateHandleInStream);
		currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream);
		restoreKVStateMetaData();
		restoreKVStateData();
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) {
			IOUtils.closeQuietly(currentStateHandleInStream);
		}
	}
}
 
Example #28
Source File: Kafka09ITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public String deserialize(byte[] messageKey, byte[] message, String topic, int partition,
	long offset) throws IOException {
	cnt++;
	DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message));
	String e = ser.deserialize(in);
	return e;
}
 
Example #29
Source File: EnumSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigurationSnapshotSerialization() throws Exception {
	EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);

	byte[] serializedConfig;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration(), serializer);
		serializedConfig = out.toByteArray();
	}

	TypeSerializerSnapshot<PublicEnum> restoredConfig;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
		restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializer);
	}

	TypeSerializerSchemaCompatibility<PublicEnum> compatResult = restoredConfig.resolveSchemaCompatibility(serializer);
	assertTrue(compatResult.isCompatibleAsIs());

	assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
	assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
	assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
	assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
	assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
	assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
	assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
}
 
Example #30
Source File: FoldApplyProcessWindowFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyWindowContext<>(windowFunction);
}