Java Code Examples for org.apache.flink.util.InstantiationUtil#deserializeObject()

The following examples show how to use org.apache.flink.util.InstantiationUtil#deserializeObject() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JobMasterTest.java From flink with Apache License 2.0

6 votes

private List<InputSplit> getRemainingInputSplits(Supplier<SerializedInputSplit> nextInputSplit) throws Exception {
	final List<InputSplit> actualInputSplits = new ArrayList<>(16);
	boolean hasMoreInputSplits = true;

	while (hasMoreInputSplits) {
		final SerializedInputSplit serializedInputSplit = nextInputSplit.get();

		if (serializedInputSplit.isEmpty()) {
			hasMoreInputSplits = false;
		} else {
			final InputSplit inputSplit = InstantiationUtil.deserializeObject(serializedInputSplit.getInputSplitData(), ClassLoader.getSystemClassLoader());

			if (inputSplit == null) {
				hasMoreInputSplits = false;
			} else {
				actualInputSplits.add(inputSplit);
			}
		}
	}

	return actualInputSplits;
}

Example 2

Source File: BaseRowSerializer.java From flink with Apache License 2.0

6 votes

@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader)
		throws IOException {
	int length = in.readInt();
	DataInputViewStream stream = new DataInputViewStream(in);
	previousTypes = new LogicalType[length];
	for (int i = 0; i < length; i++) {
		try {
			previousTypes[i] = InstantiationUtil.deserializeObject(
					stream,
					userCodeClassLoader
			);
		}
		catch (ClassNotFoundException e) {
			throw new IOException(e);
		}
	}
	this.nestedSerializersSnapshotDelegate = NestedSerializersSnapshotDelegate.readNestedSerializerSnapshots(
			in,
			userCodeClassLoader
	);
}

Example 3

Source File: StatefulComplexPayloadSerializer.java From flink with Apache License 2.0

6 votes

@Override
public ComplexPayload copy(ComplexPayload from) {
	try {
		Thread currentThread = Thread.currentThread();
		if (currentOwnerThread.compareAndSet(null, currentThread)) {
			return InstantiationUtil.deserializeObject(
				InstantiationUtil.serializeObject(from), currentThread.getContextClassLoader());
		} else {
			throw new IllegalStateException("Concurrent access to type serializer detected!");
		}
	} catch (Exception e) {
		throw new RuntimeException(e);
	} finally {
		currentOwnerThread.set(null);
	}
}

Example 4

Source File: JobManagerWatermarkTracker.java From flink with Apache License 2.0

6 votes

@Override
public long updateWatermark(long localWatermark) {
	WatermarkUpdate update = new WatermarkUpdate();
	update.id = getSubtaskId();
	update.watermark = localWatermark;
	try {
		byte[] resultBytes = aggregateManager.updateGlobalAggregate(aggregateName,
			InstantiationUtil.serializeObject(update), aggregateFunction);
		WatermarkResult result = InstantiationUtil.deserializeObject(resultBytes,
			this.getClass().getClassLoader());
		this.updateTimeoutCount += result.updateTimeoutCount;
		return result.watermark;
	} catch (ClassNotFoundException | IOException ex) {
		throw new RuntimeException(ex);
	}
}

Example 5

Source File: EnumSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSerializeEnumSerializer() throws Exception {
	EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class);

	// verify original transient parameters
	assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
	assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
	assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
	assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
	assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
	assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
	assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));

	byte[] serializedSerializer = InstantiationUtil.serializeObject(serializer);

	// deserialize and re-verify transient parameters
	serializer = InstantiationUtil.deserializeObject(serializedSerializer, Thread.currentThread().getContextClassLoader());
	assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue());
	assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue());
	assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue());
	assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue());
	assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue());
	assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue());
	assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues()));
}

Example 6

Source File: TupleSerializerConfigSnapshot.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}

Example 7

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

5 votes

private static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
	try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
			for (long offset : entry.getValue().getOffsets()) {
				in.seek(offset);
				Integer state = InstantiationUtil.
						deserializeObject(in, Thread.currentThread().getContextClassLoader());
				resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
			}
		}
	}
}

Example 8

Source File: JavaSerializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public T deserialize(DataInputView source) throws IOException {
	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(source)) {
		return InstantiationUtil.deserializeObject(
				inViewWrapper,
				Thread.currentThread().getContextClassLoader());
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not deserialize object.", e);
	}
}

Example 9

Source File: LogicalTypesTest.java From flink with Apache License 2.0

5 votes

private static void testJavaSerializability(LogicalType serializableType) {
	try {
		final LogicalType deserializedInstance = InstantiationUtil.deserializeObject(
			InstantiationUtil.serializeObject(serializableType),
			LogicalTypesTest.class.getClassLoader());

		assertEquals(serializableType, deserializedInstance);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}

Example 10

Source File: DataStructureConvertersTest.java From flink with Apache License 2.0

5 votes

private static DataStructureConverter<Object, Object> simulateSerialization(DataStructureConverter<Object, Object> converter) {
	try {
		final byte[] bytes = InstantiationUtil.serializeObject(converter);
		return InstantiationUtil.deserializeObject(bytes, DataStructureConverter.class.getClassLoader());
	} catch (Exception e) {
		throw new AssertionError("Serialization failed.", e);
	}
}

Example 11

Source File: JavaSerializer.java From flink with Apache License 2.0

5 votes

@Override
public T deserialize(DataInputView source) throws IOException {
	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(source)) {
		return InstantiationUtil.deserializeObject(
				inViewWrapper,
				Thread.currentThread().getContextClassLoader());
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not deserialize object.", e);
	}
}

Example 12

Source File: MapDataSerializer.java From flink with Apache License 2.0

5 votes

@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		DataInputViewStream inStream = new DataInputViewStream(in);
		this.previousKeyType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousValueType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousKeySerializer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousValueSerializer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
}

Example 13

Source File: FileArchivedExecutionGraphStore.java From flink with Apache License 2.0

5 votes

private ArchivedExecutionGraph loadExecutionGraph(JobID jobId) throws IOException, ClassNotFoundException {
	final File archivedExecutionGraphFile = getExecutionGraphFile(jobId);

	if (archivedExecutionGraphFile.exists()) {
		try (FileInputStream fileInputStream = new FileInputStream(archivedExecutionGraphFile)) {
			return InstantiationUtil.deserializeObject(fileInputStream, getClass().getClassLoader());
		}
	} else {
		throw new FileNotFoundException("Could not find file for archived execution graph " + jobId +
			". This indicates that the file either has been deleted or never written.");
	}
}

Example 14

Source File: CsvRowDeSerializationSchemaTest.java From Flink-CEPplus with Apache License 2.0

5 votes

private static Row deserialize(CsvRowDeserializationSchema.Builder deserSchemaBuilder, String csv) throws Exception {
	// we serialize and deserialize the schema to test runtime behavior
	// when the schema is shipped to the cluster
	final CsvRowDeserializationSchema schema = InstantiationUtil.deserializeObject(
		InstantiationUtil.serializeObject(deserSchemaBuilder.build()),
		CsvRowDeSerializationSchemaTest.class.getClassLoader());
	return schema.deserialize(csv.getBytes());
}

Example 15

Source File: SortedMapSerializerSnapshot.java From flink with Apache License 2.0

5 votes

@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		comparator = InstantiationUtil.deserializeObject(
				new DataInputViewStream(in), userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
	this.nestedSerializersSnapshotDelegate = NestedSerializersSnapshotDelegate.readNestedSerializerSnapshots(
			in,
			userCodeClassLoader);
}

Example 16

Source File: GenericArraySerializerConfigSnapshot.java From flink with Apache License 2.0

5 votes

private void readV1(DataInputView in, ClassLoader classLoader) throws IOException {
	nestedSnapshot = NestedSerializersSnapshotDelegate.legacyReadNestedSerializerSnapshots(in, classLoader);

	try (DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		componentClass = InstantiationUtil.deserializeObject(inViewWrapper, classLoader);
	}
	catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested element class in classpath.", e);
	}
}

Example 17

Source File: AbstractParameterToolTest.java From flink with Apache License 2.0

5 votes

protected void validate(ParameterTool parameter) {
	ClosureCleaner.ensureSerializable(parameter);
	internalValidate(parameter);

	// -------- test behaviour after serialization ------------
	try {
		byte[] b = InstantiationUtil.serializeObject(parameter);
		final ParameterTool copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
		internalValidate(copy);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
}

Example 18

Source File: TupleSerializerConfigSnapshot.java From flink with Apache License 2.0

5 votes

@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}

Example 19

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example 20

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}